--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:06:05.052365Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-2d1354-predictor-56dcf55695-g5tv8 namespace: kserve-ci-e2e-test resourceVersion: "20343" uid: ce3d4aa0-bd19-4b1e-8066-15eb44dfd87b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-2d1354-predictor-56dcf55695-g5tv8 to ip-10-0-136-2.ec2.internal metadata: creationTimestamp: "2026-07-02T14:06:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:06:05Z" name: isvc-init-fail-2d1354-predictor-56dcf55695-g5tv8.18be7e1ce50dec78 namespace: kserve-ci-e2e-test resourceVersion: "20349" uid: 346c8076-ab02-43b8-ac4f-81f2fa2cd9c0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8899f5b4d-w9vws source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-2d1354-predictor-56dcf55695-g5tv8 namespace: kserve-ci-e2e-test resourceVersion: "20353" uid: ce3d4aa0-bd19-4b1e-8066-15eb44dfd87b kind: Event lastTimestamp: "2026-07-02T14:06:05Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:06:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:06:05Z" name: isvc-init-fail-2d1354-predictor-56dcf55695-g5tv8.18be7e1cfeb29f38 namespace: kserve-ci-e2e-test resourceVersion: "20360" uid: d83824c8-fd04-49e6-abe1-fbe229f492f2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:06:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-2d1354-predictor-56dcf55695-g5tv8 namespace: kserve-ci-e2e-test resourceVersion: "20344" uid: ce3d4aa0-bd19-4b1e-8066-15eb44dfd87b kind: Event lastTimestamp: "2026-07-02T14:06:10Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd2b86b0a2c9500aabe87a633c3b9bab30a880b19ff42859e99cc17f113ecdb0" already present on machine metadata: creationTimestamp: "2026-07-02T14:06:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:10Z" name: isvc-init-fail-2d1354-predictor-56dcf55695-g5tv8.18be7e1d0020baeb namespace: kserve-ci-e2e-test resourceVersion: "20406" uid: 46445567-7f7e-4365-8877-1be474b6a5ab reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:06:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-2d1354-predictor-56dcf55695-g5tv8 namespace: kserve-ci-e2e-test resourceVersion: "20344" uid: ce3d4aa0-bd19-4b1e-8066-15eb44dfd87b kind: Event lastTimestamp: "2026-07-02T14:06:10Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:06:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:10Z" name: isvc-init-fail-2d1354-predictor-56dcf55695-g5tv8.18be7e1d031b5309 namespace: kserve-ci-e2e-test resourceVersion: "20410" uid: 2c9239e7-fccd-476f-bab0-7661c889f61b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:06:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-2d1354-predictor-56dcf55695-g5tv8 namespace: kserve-ci-e2e-test resourceVersion: "20344" uid: ce3d4aa0-bd19-4b1e-8066-15eb44dfd87b kind: Event lastTimestamp: "2026-07-02T14:06:10Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:06:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:10Z" name: isvc-init-fail-2d1354-predictor-56dcf55695-g5tv8.18be7e1d038b3e22 namespace: kserve-ci-e2e-test resourceVersion: "20412" uid: 35f55735-00c1-4479-bf48-93f1d8365f36 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-2d1354-predictor-56dcf55695-g5tv8 namespace: kserve-ci-e2e-test resourceVersion: "20344" uid: ce3d4aa0-bd19-4b1e-8066-15eb44dfd87b kind: Event lastTimestamp: "2026-07-02T14:06:15Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-07-02T14:06:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:15Z" name: isvc-init-fail-2d1354-predictor-56dcf55695-g5tv8.18be7e1f38e3db52 namespace: kserve-ci-e2e-test resourceVersion: "20462" uid: b7bfd138-85c6-43f4-8998-474b9c87f935 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-2d1354-predictor-56dcf55695 namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: b7bf05f1-97b5-4105-bb83-9f89a64facd0 kind: Event lastTimestamp: "2026-07-02T14:06:05Z" message: 'Created pod: isvc-init-fail-2d1354-predictor-56dcf55695-g5tv8' metadata: creationTimestamp: "2026-07-02T14:06:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:06:05Z" name: isvc-init-fail-2d1354-predictor-56dcf55695.18be7e1ce4c2da10 namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: 16bf0313-82aa-4dc5-9a41-c748fa4b7621 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-2d1354-predictor namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: d396f3f3-52fc-447e-87d6-48319755c253 kind: Event lastTimestamp: "2026-07-02T14:06:04Z" message: Scaled up replica set isvc-init-fail-2d1354-predictor-56dcf55695 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:06:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:06:04Z" name: isvc-init-fail-2d1354-predictor.18be7e1ce19b0894 namespace: kserve-ci-e2e-test resourceVersion: "20338" uid: 86a16949-e8be-4f9b-a4be-fdcad77fa6dd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T14:06:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-2d1354 namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: 86a3a5a3-bedd-4ef2-b8e3-abb6915c94dc kind: Event lastTimestamp: "2026-07-02T14:06:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:06:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:06:10Z" name: isvc-init-fail-2d1354.18be7e1ce1af929c namespace: kserve-ci-e2e-test resourceVersion: "20415" uid: 9e024226-cf52-4e97-8603-98f86e0a15b4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-2d1354 namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: 86a3a5a3-bedd-4ef2-b8e3-abb6915c94dc kind: Event lastTimestamp: "2026-07-02T14:06:05Z" message: 'Failed to update status for InferenceService "isvc-init-fail-2d1354": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-2d1354": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:06:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:06:05Z" name: isvc-init-fail-2d1354.18be7e1ce22982f4 namespace: kserve-ci-e2e-test resourceVersion: "20340" uid: 7e879bce-9d53-4d8d-a098-ba68a763f25a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-2d1354 namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: 86a3a5a3-bedd-4ef2-b8e3-abb6915c94dc kind: Event lastTimestamp: "2026-07-02T14:06:05Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-2d1354": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:06:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:06:05Z" name: isvc-init-fail-2d1354.18be7e1ce229cfc3 namespace: kserve-ci-e2e-test resourceVersion: "20342" uid: c1965619-6cf7-4b61-b5c0-a9fc65b70a12 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:59:40.928493Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n namespace: kserve-ci-e2e-test resourceVersion: "17233" uid: d195ac35-8371-4f55-96a1-f3e8b5c785a5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n to ip-10-0-136-2.ec2.internal metadata: creationTimestamp: "2026-07-02T13:59:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:59:40Z" name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n.18be7dc3757cbdce namespace: kserve-ci-e2e-test resourceVersion: "17237" uid: 6d520b22-0b52-458a-83ff-3d7ad184e49a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8899f5b4d-w9vws source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:59:41Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n namespace: kserve-ci-e2e-test resourceVersion: "17235" uid: d195ac35-8371-4f55-96a1-f3e8b5c785a5 kind: Event lastTimestamp: "2026-07-02T13:59:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-logger-raw-1b7e6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:41Z" name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n.18be7dc37d9d7b8a namespace: kserve-ci-e2e-test resourceVersion: "17255" uid: 0433b57d-37ea-4937-80c2-f49fa8a75822 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:42Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n namespace: kserve-ci-e2e-test resourceVersion: "17242" uid: d195ac35-8371-4f55-96a1-f3e8b5c785a5 kind: Event lastTimestamp: "2026-07-02T13:59:42Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:59:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:59:42Z" name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n.18be7dc3e866c1fd namespace: kserve-ci-e2e-test resourceVersion: "17268" uid: 8fc916d9-f820-4e32-af3b-461a4379eb04 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n namespace: kserve-ci-e2e-test resourceVersion: "17235" uid: d195ac35-8371-4f55-96a1-f3e8b5c785a5 kind: Event lastTimestamp: "2026-07-02T13:59:42Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd2b86b0a2c9500aabe87a633c3b9bab30a880b19ff42859e99cc17f113ecdb0" already present on machine metadata: creationTimestamp: "2026-07-02T13:59:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:42Z" name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n.18be7dc3ea391ee4 namespace: kserve-ci-e2e-test resourceVersion: "17270" uid: 83485bd0-fe52-45fd-86eb-4fb1c22dd390 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n namespace: kserve-ci-e2e-test resourceVersion: "17235" uid: d195ac35-8371-4f55-96a1-f3e8b5c785a5 kind: Event lastTimestamp: "2026-07-02T13:59:42Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T13:59:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:42Z" name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n.18be7dc3ed07c14c namespace: kserve-ci-e2e-test resourceVersion: "17271" uid: 54013723-cfc6-4586-b315-45a5cce44558 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n namespace: kserve-ci-e2e-test resourceVersion: "17235" uid: d195ac35-8371-4f55-96a1-f3e8b5c785a5 kind: Event lastTimestamp: "2026-07-02T13:59:42Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T13:59:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:42Z" name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n.18be7dc3ed691e1a namespace: kserve-ci-e2e-test resourceVersion: "17273" uid: e7d16ab7-4166-436c-901d-aeb3f220cd35 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n namespace: kserve-ci-e2e-test resourceVersion: "17235" uid: d195ac35-8371-4f55-96a1-f3e8b5c785a5 kind: Event lastTimestamp: "2026-07-02T13:59:47Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1695" already present on machine metadata: creationTimestamp: "2026-07-02T13:59:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:47Z" name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n.18be7dc50811207f namespace: kserve-ci-e2e-test resourceVersion: "17311" uid: 0e199f3f-822b-4108-a411-51fe87fd5f11 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n namespace: kserve-ci-e2e-test resourceVersion: "17235" uid: d195ac35-8371-4f55-96a1-f3e8b5c785a5 kind: Event lastTimestamp: "2026-07-02T13:59:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:59:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:47Z" name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n.18be7dc50b6bd83f namespace: kserve-ci-e2e-test resourceVersion: "17313" uid: 67e32151-9241-4845-9690-636a55a4f810 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n namespace: kserve-ci-e2e-test resourceVersion: "17235" uid: d195ac35-8371-4f55-96a1-f3e8b5c785a5 kind: Event lastTimestamp: "2026-07-02T13:59:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:59:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:47Z" name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n.18be7dc50bf95cb8 namespace: kserve-ci-e2e-test resourceVersion: "17314" uid: b8ecc680-022f-4a3f-a4a9-4570adbc88d3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n namespace: kserve-ci-e2e-test resourceVersion: "17235" uid: d195ac35-8371-4f55-96a1-f3e8b5c785a5 kind: Event lastTimestamp: "2026-07-02T13:59:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:59:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:47Z" name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n.18be7dc50c0361df namespace: kserve-ci-e2e-test resourceVersion: "17315" uid: 11c778c1-b2ef-4ee8-ab6f-a4a592c69b60 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n namespace: kserve-ci-e2e-test resourceVersion: "17235" uid: d195ac35-8371-4f55-96a1-f3e8b5c785a5 kind: Event lastTimestamp: "2026-07-02T13:59:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:59:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:47Z" name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n.18be7dc50f33054b namespace: kserve-ci-e2e-test resourceVersion: "17318" uid: 2c42e224-8080-443b-bfa8-25d041d903b3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n namespace: kserve-ci-e2e-test resourceVersion: "17235" uid: d195ac35-8371-4f55-96a1-f3e8b5c785a5 kind: Event lastTimestamp: "2026-07-02T13:59:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:59:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:47Z" name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n.18be7dc50f9b2ff4 namespace: kserve-ci-e2e-test resourceVersion: "17319" uid: 7d50934d-13fd-4785-93f6-a2423b938507 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n namespace: kserve-ci-e2e-test resourceVersion: "17235" uid: d195ac35-8371-4f55-96a1-f3e8b5c785a5 kind: Event lastTimestamp: "2026-07-02T13:59:47Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:1e7dcd5f4b9bda75b80489a614582dd16b64d12f2ecd8484f5e1c0efef78ddc5" already present on machine metadata: creationTimestamp: "2026-07-02T13:59:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:47Z" name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n.18be7dc50fa5dc2e namespace: kserve-ci-e2e-test resourceVersion: "17321" uid: 26d34907-7ee2-4aab-8a50-08b1a8413754 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n namespace: kserve-ci-e2e-test resourceVersion: "17235" uid: d195ac35-8371-4f55-96a1-f3e8b5c785a5 kind: Event lastTimestamp: "2026-07-02T13:59:47Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-02T13:59:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:47Z" name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n.18be7dc512f44d26 namespace: kserve-ci-e2e-test resourceVersion: "17322" uid: f64d3807-9498-4455-8a44-a3379191e98f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n namespace: kserve-ci-e2e-test resourceVersion: "17235" uid: d195ac35-8371-4f55-96a1-f3e8b5c785a5 kind: Event lastTimestamp: "2026-07-02T13:59:47Z" message: Started container agent metadata: creationTimestamp: "2026-07-02T13:59:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:47Z" name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n.18be7dc5135fd8b7 namespace: kserve-ci-e2e-test resourceVersion: "17323" uid: 5251e8b4-f048-49e2-a651-0b88f6e59afa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T13:59:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n namespace: kserve-ci-e2e-test resourceVersion: "17235" uid: d195ac35-8371-4f55-96a1-f3e8b5c785a5 kind: Event lastTimestamp: "2026-07-02T14:01:25Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:59:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:01:25Z" name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n.18be7dc54439a4b0 namespace: kserve-ci-e2e-test resourceVersion: "18148" uid: e22f66f1-435b-4c76-98d1-ceec3b9b39d2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T13:59:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n namespace: kserve-ci-e2e-test resourceVersion: "17235" uid: d195ac35-8371-4f55-96a1-f3e8b5c785a5 kind: Event lastTimestamp: "2026-07-02T14:01:25Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T13:59:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:01:25Z" name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n.18be7dc580240b7c namespace: kserve-ci-e2e-test resourceVersion: "18149" uid: 2cb1bdd6-0dd3-4e67-8ca7-a0055838dbbf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:01:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n namespace: kserve-ci-e2e-test resourceVersion: "17235" uid: d195ac35-8371-4f55-96a1-f3e8b5c785a5 kind: Event lastTimestamp: "2026-07-02T14:01:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:01:16Z" name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n.18be7dd9a34a6989 namespace: kserve-ci-e2e-test resourceVersion: "18009" uid: 2e6e6a83-ca36-42bd-b547-9ee255896d81 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:01:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n namespace: kserve-ci-e2e-test resourceVersion: "17235" uid: d195ac35-8371-4f55-96a1-f3e8b5c785a5 kind: Event lastTimestamp: "2026-07-02T14:01:16Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-02T14:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:01:16Z" name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n.18be7dd9a34b29a5 namespace: kserve-ci-e2e-test resourceVersion: "18013" uid: 766b8006-9ffb-4b25-8c5a-66fe87aacf6d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:01:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n namespace: kserve-ci-e2e-test resourceVersion: "17235" uid: d195ac35-8371-4f55-96a1-f3e8b5c785a5 kind: Event lastTimestamp: "2026-07-02T14:01:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:01:16Z" name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n.18be7dd9a34b5cc0 namespace: kserve-ci-e2e-test resourceVersion: "18015" uid: 941a29be-543f-4cbf-a79f-f3d95f894619 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n namespace: kserve-ci-e2e-test resourceVersion: "17235" uid: d195ac35-8371-4f55-96a1-f3e8b5c785a5 kind: Event lastTimestamp: "2026-07-02T14:01:25Z" message: 'Readiness probe failed: Get "https://10.133.0.33:8643/healthz": dial tcp 10.133.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:01:25Z" name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n.18be7ddab05f8ee7 namespace: kserve-ci-e2e-test resourceVersion: "18147" uid: 47e3b66d-33f1-4d0a-a5d7-22bb889f90fa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75 namespace: kserve-ci-e2e-test resourceVersion: "17222" uid: a6939172-46ae-4c10-9d56-c02e5ad3cb40 kind: Event lastTimestamp: "2026-07-02T13:59:40Z" message: 'Created pod: isvc-logger-raw-1b7e6-predictor-7ff7d59b75-fpd2n' metadata: creationTimestamp: "2026-07-02T13:59:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:59:40Z" name: isvc-logger-raw-1b7e6-predictor-7ff7d59b75.18be7dc37502ada7 namespace: kserve-ci-e2e-test resourceVersion: "17234" uid: a991afc5-923e-455d-a5d8-d98eee88742c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-1b7e6-predictor namespace: kserve-ci-e2e-test resourceVersion: "17220" uid: 46311ccc-3dc9-4f61-8523-39096157b05f kind: Event lastTimestamp: "2026-07-02T13:59:40Z" message: Scaled up replica set isvc-logger-raw-1b7e6-predictor-7ff7d59b75 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:59:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:59:40Z" name: isvc-logger-raw-1b7e6-predictor.18be7dc37106ddaf namespace: kserve-ci-e2e-test resourceVersion: "17223" uid: 7ad29751-d178-4fb6-9022-35a41ee3b8b4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:59:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-1b7e6-predictor namespace: kserve-ci-e2e-test resourceVersion: "17221" uid: 7acd5181-4a72-4697-8726-5d89bc2c6612 kind: Event lastTimestamp: "2026-07-02T14:00:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:59:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:00:25Z" name: isvc-logger-raw-1b7e6-predictor.18be7dc6ef8b4f11 namespace: kserve-ci-e2e-test resourceVersion: "17616" uid: c28450db-8501-48c4-87d6-c4273ee905cf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:59:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-1b7e6-predictor namespace: kserve-ci-e2e-test resourceVersion: "17221" uid: 7acd5181-4a72-4697-8726-5d89bc2c6612 kind: Event lastTimestamp: "2026-07-02T14:00:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:59:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:00:25Z" name: isvc-logger-raw-1b7e6-predictor.18be7dc6eff43a90 namespace: kserve-ci-e2e-test resourceVersion: "17618" uid: 597a4671-eb65-4995-925e-1c69760e916f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:00:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-1b7e6-predictor namespace: kserve-ci-e2e-test resourceVersion: "17396" uid: 7acd5181-4a72-4697-8726-5d89bc2c6612 kind: Event lastTimestamp: "2026-07-02T14:01:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:00:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:01:10Z" name: isvc-logger-raw-1b7e6-predictor.18be7dca6effd8b9 namespace: kserve-ci-e2e-test resourceVersion: "17937" uid: 0df67907-e70a-498c-a51e-f8d65648c247 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:00:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-1b7e6-predictor namespace: kserve-ci-e2e-test resourceVersion: "17396" uid: 7acd5181-4a72-4697-8726-5d89bc2c6612 kind: Event lastTimestamp: "2026-07-02T14:01:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:00:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:01:10Z" name: isvc-logger-raw-1b7e6-predictor.18be7dca6f63206d namespace: kserve-ci-e2e-test resourceVersion: "17938" uid: a3d2f8c2-8ee6-4754-bc77-74d69cad55a8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T13:59:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-1b7e6 namespace: kserve-ci-e2e-test resourceVersion: "17211" uid: 3778cb68-8845-47e5-a18a-6119d551464f kind: Event lastTimestamp: "2026-07-02T14:01:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:59:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:01:05Z" name: isvc-logger-raw-1b7e6.18be7dc3710d0e62 namespace: kserve-ci-e2e-test resourceVersion: "17903" uid: f6737eb1-eb4e-43ea-947b-2c2539f8b17a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:59:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-1b7e6 namespace: kserve-ci-e2e-test resourceVersion: "17211" uid: 3778cb68-8845-47e5-a18a-6119d551464f kind: Event lastTimestamp: "2026-07-02T13:59:40Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-1b7e6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-1b7e6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:59:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:59:40Z" name: isvc-logger-raw-1b7e6.18be7dc371a64727 namespace: kserve-ci-e2e-test resourceVersion: "17244" uid: 29fe30eb-fb15-43d1-849a-00872d903dfc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-1b7e6 namespace: kserve-ci-e2e-test resourceVersion: "17215" uid: 3778cb68-8845-47e5-a18a-6119d551464f kind: Event lastTimestamp: "2026-07-02T13:59:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-1b7e6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:59:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:59:40Z" name: isvc-logger-raw-1b7e6.18be7dc378718ac6 namespace: kserve-ci-e2e-test resourceVersion: "17245" uid: d139330c-45ad-459c-88ab-ef260c149f9f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:01:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-1b7e6 namespace: kserve-ci-e2e-test resourceVersion: "17901" uid: 3778cb68-8845-47e5-a18a-6119d551464f kind: Event lastTimestamp: "2026-07-02T14:01:05Z" message: InferenceService [isvc-logger-raw-1b7e6] is Ready metadata: creationTimestamp: "2026-07-02T14:01:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:01:05Z" name: isvc-logger-raw-1b7e6.18be7dd737e9cceb namespace: kserve-ci-e2e-test resourceVersion: "17902" uid: b4cece48-b6fb-4bb2-b65d-69d8bb235035 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:04:11.608878Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58 namespace: kserve-ci-e2e-test resourceVersion: "19364" uid: 96ed8409-2230-4cd3-94b8-3740dae79b3a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58 to ip-10-0-136-2.ec2.internal metadata: creationTimestamp: "2026-07-02T14:04:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:04:11Z" name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58.18be7e027b4bb336 namespace: kserve-ci-e2e-test resourceVersion: "19368" uid: 5454f6c3-887f-4aca-bd38-76e8aa132b73 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8899f5b4d-w9vws source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:12Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58 namespace: kserve-ci-e2e-test resourceVersion: "19375" uid: 96ed8409-2230-4cd3-94b8-3740dae79b3a kind: Event lastTimestamp: "2026-07-02T14:04:12Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:04:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:04:12Z" name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58.18be7e02950c2231 namespace: kserve-ci-e2e-test resourceVersion: "19383" uid: 4ae75632-c8fc-4e95-bc6a-f0eeb245b666 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58 namespace: kserve-ci-e2e-test resourceVersion: "19366" uid: 96ed8409-2230-4cd3-94b8-3740dae79b3a kind: Event lastTimestamp: "2026-07-02T14:04:12Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd2b86b0a2c9500aabe87a633c3b9bab30a880b19ff42859e99cc17f113ecdb0" already present on machine metadata: creationTimestamp: "2026-07-02T14:04:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:12Z" name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58.18be7e0296df0661 namespace: kserve-ci-e2e-test resourceVersion: "19386" uid: fc3293d0-2038-4778-91c1-ca6497cacba4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58 namespace: kserve-ci-e2e-test resourceVersion: "19366" uid: 96ed8409-2230-4cd3-94b8-3740dae79b3a kind: Event lastTimestamp: "2026-07-02T14:04:12Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:04:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:12Z" name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58.18be7e029a7d447c namespace: kserve-ci-e2e-test resourceVersion: "19389" uid: cddc53bb-c683-4b5f-88dc-8ce9028500cd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58 namespace: kserve-ci-e2e-test resourceVersion: "19366" uid: 96ed8409-2230-4cd3-94b8-3740dae79b3a kind: Event lastTimestamp: "2026-07-02T14:04:12Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:04:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:12Z" name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58.18be7e029af60a31 namespace: kserve-ci-e2e-test resourceVersion: "19390" uid: 94e04e1d-b12c-4a20-99a6-829f1707ac03 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58 namespace: kserve-ci-e2e-test resourceVersion: "19366" uid: 96ed8409-2230-4cd3-94b8-3740dae79b3a kind: Event lastTimestamp: "2026-07-02T14:04:15Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1695" already present on machine metadata: creationTimestamp: "2026-07-02T14:04:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:15Z" name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58.18be7e037eaca4a8 namespace: kserve-ci-e2e-test resourceVersion: "19419" uid: 57523e40-5e05-4d50-9fb8-7d3557d23c15 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58 namespace: kserve-ci-e2e-test resourceVersion: "19366" uid: 96ed8409-2230-4cd3-94b8-3740dae79b3a kind: Event lastTimestamp: "2026-07-02T14:04:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:16Z" name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58.18be7e0381d5847b namespace: kserve-ci-e2e-test resourceVersion: "19421" uid: ea78e1e3-f241-4fbc-b8a5-fd71c1b17ef8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58 namespace: kserve-ci-e2e-test resourceVersion: "19366" uid: 96ed8409-2230-4cd3-94b8-3740dae79b3a kind: Event lastTimestamp: "2026-07-02T14:04:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:16Z" name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58.18be7e03824701b6 namespace: kserve-ci-e2e-test resourceVersion: "19422" uid: 1af4bbe0-8399-41f7-b70f-6c8e1620500c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58 namespace: kserve-ci-e2e-test resourceVersion: "19366" uid: 96ed8409-2230-4cd3-94b8-3740dae79b3a kind: Event lastTimestamp: "2026-07-02T14:04:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:16Z" name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58.18be7e038251472f namespace: kserve-ci-e2e-test resourceVersion: "19426" uid: 95ff4200-76c0-446f-be1d-a2811a5539cc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58 namespace: kserve-ci-e2e-test resourceVersion: "19366" uid: 96ed8409-2230-4cd3-94b8-3740dae79b3a kind: Event lastTimestamp: "2026-07-02T14:04:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:16Z" name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58.18be7e03857ebd62 namespace: kserve-ci-e2e-test resourceVersion: "19428" uid: 774026b3-6527-495b-89b5-59d7628da0ae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58 namespace: kserve-ci-e2e-test resourceVersion: "19366" uid: 96ed8409-2230-4cd3-94b8-3740dae79b3a kind: Event lastTimestamp: "2026-07-02T14:04:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:16Z" name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58.18be7e0385f50186 namespace: kserve-ci-e2e-test resourceVersion: "19429" uid: a7354504-0b08-409b-a2c7-923dc491067b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T14:04:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58 namespace: kserve-ci-e2e-test resourceVersion: "19366" uid: 96ed8409-2230-4cd3-94b8-3740dae79b3a kind: Event lastTimestamp: "2026-07-02T14:06:03Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:04:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:03Z" name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58.18be7e03f7214683 namespace: kserve-ci-e2e-test resourceVersion: "20305" uid: 3c5f7f1a-4d30-4bbc-a1f7-41aa4d179851 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58 namespace: kserve-ci-e2e-test resourceVersion: "19366" uid: 96ed8409-2230-4cd3-94b8-3740dae79b3a kind: Event lastTimestamp: "2026-07-02T14:05:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:59Z" name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58.18be7e1bb3cfae63 namespace: kserve-ci-e2e-test resourceVersion: "20272" uid: ddd4352f-710a-48f1-859a-f5c90e7527de reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58 namespace: kserve-ci-e2e-test resourceVersion: "19366" uid: 96ed8409-2230-4cd3-94b8-3740dae79b3a kind: Event lastTimestamp: "2026-07-02T14:05:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:59Z" name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58.18be7e1bb3d04193 namespace: kserve-ci-e2e-test resourceVersion: "20273" uid: d7ccd771-e79d-4d15-adfa-33479ec97208 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58 namespace: kserve-ci-e2e-test resourceVersion: "19366" uid: 96ed8409-2230-4cd3-94b8-3740dae79b3a kind: Event lastTimestamp: "2026-07-02T14:06:03Z" message: 'Readiness probe failed: Get "https://10.133.0.35:8643/healthz": dial tcp 10.133.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:06:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:03Z" name: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58.18be7e1ca572a7dc namespace: kserve-ci-e2e-test resourceVersion: "20304" uid: 92390614-6eec-46f9-b1d7-97c6c2d689a0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-6f2d5c-predictor-8588fb45dd namespace: kserve-ci-e2e-test resourceVersion: "19355" uid: 2e438638-8ced-4787-ba1a-b198c80e8e44 kind: Event lastTimestamp: "2026-07-02T14:04:11Z" message: 'Created pod: isvc-primary-6f2d5c-predictor-8588fb45dd-wcq58' metadata: creationTimestamp: "2026-07-02T14:04:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:04:11Z" name: isvc-primary-6f2d5c-predictor-8588fb45dd.18be7e027ad9bca9 namespace: kserve-ci-e2e-test resourceVersion: "19365" uid: 441aaccb-755a-4e04-bccb-b9305b5d5a99 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-6f2d5c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19353" uid: dc2f7ae4-aa94-48c5-ab06-2e8d417680c7 kind: Event lastTimestamp: "2026-07-02T14:04:11Z" message: Scaled up replica set isvc-primary-6f2d5c-predictor-8588fb45dd from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:04:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:04:11Z" name: isvc-primary-6f2d5c-predictor.18be7e027577e88f namespace: kserve-ci-e2e-test resourceVersion: "19357" uid: 71f705dc-f9e6-4ebb-90c4-dd70ffb002d5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:04:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-6f2d5c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19354" uid: 8c23e1ce-9a8c-4c42-95a1-2ebb7b89c33c kind: Event lastTimestamp: "2026-07-02T14:04:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:04:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:04:56Z" name: isvc-primary-6f2d5c-predictor.18be7e05f3cc1c17 namespace: kserve-ci-e2e-test resourceVersion: "19722" uid: b5d5f433-85f2-4426-b00f-b72332b1c4bd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:04:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-6f2d5c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19354" uid: 8c23e1ce-9a8c-4c42-95a1-2ebb7b89c33c kind: Event lastTimestamp: "2026-07-02T14:04:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:04:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:04:56Z" name: isvc-primary-6f2d5c-predictor.18be7e05f4327b1b namespace: kserve-ci-e2e-test resourceVersion: "19723" uid: 500388fe-5342-47f2-a4f2-258d08b92cd9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:04:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-6f2d5c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19511" uid: 8c23e1ce-9a8c-4c42-95a1-2ebb7b89c33c kind: Event lastTimestamp: "2026-07-02T14:05:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:04:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:56Z" name: isvc-primary-6f2d5c-predictor.18be7e097363b42f namespace: kserve-ci-e2e-test resourceVersion: "20215" uid: c40bcd90-cd65-4727-854f-1645b3248070 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:04:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-6f2d5c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19511" uid: 8c23e1ce-9a8c-4c42-95a1-2ebb7b89c33c kind: Event lastTimestamp: "2026-07-02T14:05:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:04:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:56Z" name: isvc-primary-6f2d5c-predictor.18be7e0973ce4338 namespace: kserve-ci-e2e-test resourceVersion: "20216" uid: 2c1d8e79-1338-4181-9aaf-68de0e37e8b9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T14:04:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-6f2d5c namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 8802f408-e86a-42bb-a7f1-c7e4fe4d71fb kind: Event lastTimestamp: "2026-07-02T14:05:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:04:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:05:34Z" name: isvc-primary-6f2d5c.18be7e02756ff0e9 namespace: kserve-ci-e2e-test resourceVersion: "19990" uid: 0026a774-c476-4d12-9dd8-a1b5c58f7a85 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:04:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-6f2d5c namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 8802f408-e86a-42bb-a7f1-c7e4fe4d71fb kind: Event lastTimestamp: "2026-07-02T14:04:11Z" message: 'Failed to update status for InferenceService "isvc-primary-6f2d5c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-6f2d5c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:04:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:04:11Z" name: isvc-primary-6f2d5c.18be7e02762036cc namespace: kserve-ci-e2e-test resourceVersion: "19374" uid: 45d4fc25-f15f-465e-9602-5adc92f6797e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:04:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-6f2d5c namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 8802f408-e86a-42bb-a7f1-c7e4fe4d71fb kind: Event lastTimestamp: "2026-07-02T14:04:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-6f2d5c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:04:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:04:11Z" name: isvc-primary-6f2d5c.18be7e0276209b19 namespace: kserve-ci-e2e-test resourceVersion: "19376" uid: ca45735d-9923-44a3-960e-7be3f1833aeb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-6f2d5c namespace: kserve-ci-e2e-test resourceVersion: "19988" uid: 8802f408-e86a-42bb-a7f1-c7e4fe4d71fb kind: Event lastTimestamp: "2026-07-02T14:05:34Z" message: InferenceService [isvc-primary-6f2d5c] is Ready metadata: creationTimestamp: "2026-07-02T14:05:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:05:34Z" name: isvc-primary-6f2d5c.18be7e15af66a048 namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: f372b6c1-914e-4a0c-9a61-b8aa45e3cd8e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:53:55.008325Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf namespace: kserve-ci-e2e-test resourceVersion: "14004" uid: cb28b844-62f9-408a-8113-4168015658c1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf to ip-10-0-136-2.ec2.internal metadata: creationTimestamp: "2026-07-02T13:53:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:53:55Z" name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf.18be7d72eb09da94 namespace: kserve-ci-e2e-test resourceVersion: "14013" uid: 4f28900c-6efb-4d7c-8abe-ac4471040f4e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8899f5b4d-w9vws source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:55Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: cb28b844-62f9-408a-8113-4168015658c1 kind: Event lastTimestamp: "2026-07-02T13:53:55Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-raw-sklearn-batcher-2cb97-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:55Z" name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf.18be7d72f7e84f23 namespace: kserve-ci-e2e-test resourceVersion: "14025" uid: b47e260c-c8f5-4040-ad37-d212e3348a0d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:56Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf namespace: kserve-ci-e2e-test resourceVersion: "14016" uid: cb28b844-62f9-408a-8113-4168015658c1 kind: Event lastTimestamp: "2026-07-02T13:53:56Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:53:56Z" name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf.18be7d73275dc8b2 namespace: kserve-ci-e2e-test resourceVersion: "14037" uid: 25a46700-bf05-4bf2-8a5e-76e3701e2f01 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: cb28b844-62f9-408a-8113-4168015658c1 kind: Event lastTimestamp: "2026-07-02T13:53:56Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd2b86b0a2c9500aabe87a633c3b9bab30a880b19ff42859e99cc17f113ecdb0" already present on machine metadata: creationTimestamp: "2026-07-02T13:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:56Z" name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf.18be7d732907e220 namespace: kserve-ci-e2e-test resourceVersion: "14040" uid: 45e804f2-65bd-424e-8415-056c7bd82399 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: cb28b844-62f9-408a-8113-4168015658c1 kind: Event lastTimestamp: "2026-07-02T13:53:56Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T13:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:56Z" name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf.18be7d732c051906 namespace: kserve-ci-e2e-test resourceVersion: "14042" uid: db9eec67-4967-4160-ae94-41f57c45aa7d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: cb28b844-62f9-408a-8113-4168015658c1 kind: Event lastTimestamp: "2026-07-02T13:53:56Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T13:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:56Z" name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf.18be7d732c7195af namespace: kserve-ci-e2e-test resourceVersion: "14043" uid: 8de8047f-20df-4294-ac0c-94f6abb0cb05 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: cb28b844-62f9-408a-8113-4168015658c1 kind: Event lastTimestamp: "2026-07-02T13:53:59Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1695" metadata: creationTimestamp: "2026-07-02T13:53:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:59Z" name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf.18be7d7413bf0d00 namespace: kserve-ci-e2e-test resourceVersion: "14076" uid: 9afb7042-1b5d-4f03-92ab-cc7eabf7b426 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: cb28b844-62f9-408a-8113-4168015658c1 kind: Event lastTimestamp: "2026-07-02T13:54:12Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1695" in 12.838s (12.838s including waiting). Image size: 1555877318 bytes.' metadata: creationTimestamp: "2026-07-02T13:54:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:12Z" name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf.18be7d7710f54825 namespace: kserve-ci-e2e-test resourceVersion: "14171" uid: bcf6e8d7-4c7c-4c45-9569-83397d5b5316 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: cb28b844-62f9-408a-8113-4168015658c1 kind: Event lastTimestamp: "2026-07-02T13:54:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:54:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:12Z" name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf.18be7d7714157b72 namespace: kserve-ci-e2e-test resourceVersion: "14173" uid: f4eb8c59-e3a1-4f4a-97b2-008d6cd15dbd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: cb28b844-62f9-408a-8113-4168015658c1 kind: Event lastTimestamp: "2026-07-02T13:54:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:54:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:12Z" name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf.18be7d771484c066 namespace: kserve-ci-e2e-test resourceVersion: "14174" uid: 66716a14-1486-4285-bd69-ca1ce4c7fe05 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: cb28b844-62f9-408a-8113-4168015658c1 kind: Event lastTimestamp: "2026-07-02T13:54:12Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T13:54:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:12Z" name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf.18be7d7714916bcf namespace: kserve-ci-e2e-test resourceVersion: "14175" uid: 7366f2a5-e0f9-443e-a11f-52340a85f5b7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: cb28b844-62f9-408a-8113-4168015658c1 kind: Event lastTimestamp: "2026-07-02T13:54:15Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.248s (2.248s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T13:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:15Z" name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf.18be7d779a9abaa3 namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: 77921abc-9daa-400e-bb1f-7ec79676a3c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: cb28b844-62f9-408a-8113-4168015658c1 kind: Event lastTimestamp: "2026-07-02T13:54:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:15Z" name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf.18be7d779e7cd993 namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: af40bf7d-eb48-4cc9-99be-511856eff33f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: cb28b844-62f9-408a-8113-4168015658c1 kind: Event lastTimestamp: "2026-07-02T13:54:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:15Z" name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf.18be7d779eeb9ea0 namespace: kserve-ci-e2e-test resourceVersion: "14194" uid: d3a3151d-f47f-49f3-8817-b1d33e749076 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: cb28b844-62f9-408a-8113-4168015658c1 kind: Event lastTimestamp: "2026-07-02T13:54:15Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:1e7dcd5f4b9bda75b80489a614582dd16b64d12f2ecd8484f5e1c0efef78ddc5" metadata: creationTimestamp: "2026-07-02T13:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:15Z" name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf.18be7d779ef7812a namespace: kserve-ci-e2e-test resourceVersion: "14195" uid: 1cbfa397-3e3e-47dc-9c03-45532983465f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: cb28b844-62f9-408a-8113-4168015658c1 kind: Event lastTimestamp: "2026-07-02T13:54:17Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:1e7dcd5f4b9bda75b80489a614582dd16b64d12f2ecd8484f5e1c0efef78ddc5" in 2.5s (2.5s including waiting). Image size: 239994521 bytes.' metadata: creationTimestamp: "2026-07-02T13:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:17Z" name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf.18be7d7834070372 namespace: kserve-ci-e2e-test resourceVersion: "14213" uid: 8132abac-995c-4400-9ac1-dd0da282c7a1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: cb28b844-62f9-408a-8113-4168015658c1 kind: Event lastTimestamp: "2026-07-02T13:54:17Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-02T13:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:17Z" name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf.18be7d78373f454a namespace: kserve-ci-e2e-test resourceVersion: "14215" uid: 10fe86a5-ed71-4132-81ed-e5da7558ec5a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: cb28b844-62f9-408a-8113-4168015658c1 kind: Event lastTimestamp: "2026-07-02T13:54:17Z" message: Started container agent metadata: creationTimestamp: "2026-07-02T13:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:17Z" name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf.18be7d7837ab16c0 namespace: kserve-ci-e2e-test resourceVersion: "14216" uid: 816c8507-fd41-4745-896b-a95e68f3e410 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T13:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: cb28b844-62f9-408a-8113-4168015658c1 kind: Event lastTimestamp: "2026-07-02T13:55:51Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:51Z" name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf.18be7d784af4f64f namespace: kserve-ci-e2e-test resourceVersion: "15031" uid: 88f695cf-36d7-483f-b293-2537c1f7fc81 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T13:54:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: cb28b844-62f9-408a-8113-4168015658c1 kind: Event lastTimestamp: "2026-07-02T13:55:41Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T13:54:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:41Z" name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf.18be7d7886d47652 namespace: kserve-ci-e2e-test resourceVersion: "14934" uid: fc23adc9-754d-4a11-9117-592881a18d6b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: cb28b844-62f9-408a-8113-4168015658c1 kind: Event lastTimestamp: "2026-07-02T13:55:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:40Z" name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf.18be7d8b66113f2d namespace: kserve-ci-e2e-test resourceVersion: "14879" uid: e620c241-1091-4cef-9ee7-0f03e662092f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: cb28b844-62f9-408a-8113-4168015658c1 kind: Event lastTimestamp: "2026-07-02T13:55:40Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-02T13:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:40Z" name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf.18be7d8b661174c0 namespace: kserve-ci-e2e-test resourceVersion: "14875" uid: e569664d-ee85-4405-a34f-fa28c36da834 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: cb28b844-62f9-408a-8113-4168015658c1 kind: Event lastTimestamp: "2026-07-02T13:55:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:40Z" name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf.18be7d8b66b296cd namespace: kserve-ci-e2e-test resourceVersion: "14884" uid: a70b5c14-e54d-4a13-bb95-bcac3ce2cafa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:55:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: cb28b844-62f9-408a-8113-4168015658c1 kind: Event lastTimestamp: "2026-07-02T13:55:50Z" message: 'Readiness probe failed: Get "https://10.133.0.27:8643/healthz": dial tcp 10.133.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:50Z" name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf.18be7d8c8d0828c2 namespace: kserve-ci-e2e-test resourceVersion: "15025" uid: 849d96a0-18aa-4262-b6fb-a4794bc0bea2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457 namespace: kserve-ci-e2e-test resourceVersion: "13993" uid: d48bb213-6cc0-4dde-b786-a946970c608e kind: Event lastTimestamp: "2026-07-02T13:53:54Z" message: 'Created pod: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457-px4pf' metadata: creationTimestamp: "2026-07-02T13:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:53:54Z" name: isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457.18be7d72e8d32bc4 namespace: kserve-ci-e2e-test resourceVersion: "14006" uid: 993bb9e4-1dc3-4fce-811d-4d715dd7b39e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-2cb97-predictor namespace: kserve-ci-e2e-test resourceVersion: "13992" uid: c9a5abc0-e7a3-4c57-8f62-71c45309e5d8 kind: Event lastTimestamp: "2026-07-02T13:53:54Z" message: Scaled up replica set isvc-raw-sklearn-batcher-2cb97-predictor-75484bf457 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:53:54Z" name: isvc-raw-sklearn-batcher-2cb97-predictor.18be7d72def4026b namespace: kserve-ci-e2e-test resourceVersion: "13994" uid: 6dbce1ba-68a8-46fe-962f-7c2fb38ff6bf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:54:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-2cb97-predictor namespace: kserve-ci-e2e-test resourceVersion: "14000" uid: 98e52727-12bd-4adc-a5a6-ca2052545dfd kind: Event lastTimestamp: "2026-07-02T13:54:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:54:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:54:39Z" name: isvc-raw-sklearn-batcher-2cb97-predictor.18be7d766430cbdd namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: 9a7c7b11-13e8-4105-9332-07cdb9f7a29b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:54:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-2cb97-predictor namespace: kserve-ci-e2e-test resourceVersion: "14000" uid: 98e52727-12bd-4adc-a5a6-ca2052545dfd kind: Event lastTimestamp: "2026-07-02T13:54:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:54:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:54:39Z" name: isvc-raw-sklearn-batcher-2cb97-predictor.18be7d766494bf2d namespace: kserve-ci-e2e-test resourceVersion: "14384" uid: f7838209-1668-4cee-99f2-36a139c3a39a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:54:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-2cb97-predictor namespace: kserve-ci-e2e-test resourceVersion: "14148" uid: 98e52727-12bd-4adc-a5a6-ca2052545dfd kind: Event lastTimestamp: "2026-07-02T13:55:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:54:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:55:40Z" name: isvc-raw-sklearn-batcher-2cb97-predictor.18be7d80e1d40014 namespace: kserve-ci-e2e-test resourceVersion: "14835" uid: 17b7191a-844d-4a4f-923b-8fd64c25c7c6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:54:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-2cb97-predictor namespace: kserve-ci-e2e-test resourceVersion: "14148" uid: 98e52727-12bd-4adc-a5a6-ca2052545dfd kind: Event lastTimestamp: "2026-07-02T13:55:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:54:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:55:40Z" name: isvc-raw-sklearn-batcher-2cb97-predictor.18be7d80e2403ff0 namespace: kserve-ci-e2e-test resourceVersion: "14837" uid: 99ede5d8-0206-4bff-a38f-e6adc119a45f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-07-02T13:53:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-2cb97 namespace: kserve-ci-e2e-test resourceVersion: "13984" uid: 98ace4a7-624d-49be-be77-3452ea5365ee kind: Event lastTimestamp: "2026-07-02T13:55:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:55:31Z" name: isvc-raw-sklearn-batcher-2cb97.18be7d72e4f74d9a namespace: kserve-ci-e2e-test resourceVersion: "14763" uid: 726855a4-f0d5-4359-b1c1-5996337e895e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:53:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-2cb97 namespace: kserve-ci-e2e-test resourceVersion: "13984" uid: 98ace4a7-624d-49be-be77-3452ea5365ee kind: Event lastTimestamp: "2026-07-02T13:53:55Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-2cb97": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-2cb97": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:53:55Z" name: isvc-raw-sklearn-batcher-2cb97.18be7d72e54e8e4e namespace: kserve-ci-e2e-test resourceVersion: "14019" uid: 8b8fc84b-af13-4e1e-b8d0-2c5bdfd33f08 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:53:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-2cb97 namespace: kserve-ci-e2e-test resourceVersion: "13984" uid: 98ace4a7-624d-49be-be77-3452ea5365ee kind: Event lastTimestamp: "2026-07-02T13:53:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-2cb97": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:53:55Z" name: isvc-raw-sklearn-batcher-2cb97.18be7d72e54ee35a namespace: kserve-ci-e2e-test resourceVersion: "14020" uid: a6b1bc1e-a2cd-4cf4-bac3-4224ce8994f1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-2cb97 namespace: kserve-ci-e2e-test resourceVersion: "14761" uid: 98ace4a7-624d-49be-be77-3452ea5365ee kind: Event lastTimestamp: "2026-07-02T13:55:31Z" message: InferenceService [isvc-raw-sklearn-batcher-2cb97] is Ready metadata: creationTimestamp: "2026-07-02T13:55:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:55:31Z" name: isvc-raw-sklearn-batcher-2cb97.18be7d89527b7e76 namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: 2e255ea9-b994-4baf-ac51-bb43f1b427b1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:05:41.826020Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-6f2d5c-predictor-665b6cbc47-6jrkj namespace: kserve-ci-e2e-test resourceVersion: "20074" uid: 074459bf-3492-4437-ab49-09a6381701cf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-6f2d5c-predictor-665b6cbc47-6jrkj to ip-10-0-136-2.ec2.internal metadata: creationTimestamp: "2026-07-02T14:05:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:05:41Z" name: isvc-secondary-6f2d5c-predictor-665b6cbc47-6jrkj.18be7e177ca8f050 namespace: kserve-ci-e2e-test resourceVersion: "20078" uid: 73a7d8f2-d17e-4353-aac0-ce862744018d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8899f5b4d-w9vws source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:42Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-6f2d5c-predictor-665b6cbc47-6jrkj namespace: kserve-ci-e2e-test resourceVersion: "20083" uid: 074459bf-3492-4437-ab49-09a6381701cf kind: Event lastTimestamp: "2026-07-02T14:05:42Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:05:42Z" name: isvc-secondary-6f2d5c-predictor-665b6cbc47-6jrkj.18be7e1797028cf3 namespace: kserve-ci-e2e-test resourceVersion: "20093" uid: 53d33247-1d16-4fff-bffe-13ebbd42f72a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-6f2d5c-predictor-665b6cbc47-6jrkj namespace: kserve-ci-e2e-test resourceVersion: "20076" uid: 074459bf-3492-4437-ab49-09a6381701cf kind: Event lastTimestamp: "2026-07-02T14:05:48Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd2b86b0a2c9500aabe87a633c3b9bab30a880b19ff42859e99cc17f113ecdb0" already present on machine metadata: creationTimestamp: "2026-07-02T14:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:48Z" name: isvc-secondary-6f2d5c-predictor-665b6cbc47-6jrkj.18be7e17989c4681 namespace: kserve-ci-e2e-test resourceVersion: "20141" uid: ea472e94-c777-4613-81e9-66f93aeff6d8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-6f2d5c-predictor-665b6cbc47-6jrkj namespace: kserve-ci-e2e-test resourceVersion: "20076" uid: 074459bf-3492-4437-ab49-09a6381701cf kind: Event lastTimestamp: "2026-07-02T14:05:48Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:48Z" name: isvc-secondary-6f2d5c-predictor-665b6cbc47-6jrkj.18be7e179b310b0d namespace: kserve-ci-e2e-test resourceVersion: "20145" uid: 46cecf2a-c62b-4136-b713-8a0c586c7afd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-6f2d5c-predictor-665b6cbc47-6jrkj namespace: kserve-ci-e2e-test resourceVersion: "20076" uid: 074459bf-3492-4437-ab49-09a6381701cf kind: Event lastTimestamp: "2026-07-02T14:05:48Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:48Z" name: isvc-secondary-6f2d5c-predictor-665b6cbc47-6jrkj.18be7e179ba977ab namespace: kserve-ci-e2e-test resourceVersion: "20146" uid: 5d3e5e45-ad0c-4250-ab05-c415ffe38e75 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-6f2d5c-predictor-665b6cbc47-6jrkj namespace: kserve-ci-e2e-test resourceVersion: "20076" uid: 074459bf-3492-4437-ab49-09a6381701cf kind: Event lastTimestamp: "2026-07-02T14:05:55Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-6f2d5c-predictor-665b6cbc47-6jrkj_kserve-ci-e2e-test(074459bf-3492-4437-ab49-09a6381701cf) metadata: creationTimestamp: "2026-07-02T14:05:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:55Z" name: isvc-secondary-6f2d5c-predictor-665b6cbc47-6jrkj.18be7e1aa6a93c70 namespace: kserve-ci-e2e-test resourceVersion: "20199" uid: ffa395f3-5777-4870-b3d5-b139921866f9 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-6f2d5c-predictor-665b6cbc47 namespace: kserve-ci-e2e-test resourceVersion: "20065" uid: 167f012c-aa55-4e9f-9373-81a08743f677 kind: Event lastTimestamp: "2026-07-02T14:05:41Z" message: 'Created pod: isvc-secondary-6f2d5c-predictor-665b6cbc47-6jrkj' metadata: creationTimestamp: "2026-07-02T14:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:41Z" name: isvc-secondary-6f2d5c-predictor-665b6cbc47.18be7e177c379f7f namespace: kserve-ci-e2e-test resourceVersion: "20075" uid: c97ae875-167e-44f9-974b-5f8740dab8ca reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-6f2d5c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20063" uid: 696968ef-70d4-4dd7-9e7c-ac29b0e4cf36 kind: Event lastTimestamp: "2026-07-02T14:05:41Z" message: Scaled up replica set isvc-secondary-6f2d5c-predictor-665b6cbc47 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:41Z" name: isvc-secondary-6f2d5c-predictor.18be7e1777110179 namespace: kserve-ci-e2e-test resourceVersion: "20066" uid: b2605485-493e-4e8a-8323-f3d37712677c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-6f2d5c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20064" uid: 640d8c58-81a5-449e-97eb-a86f5289f5f8 kind: Event lastTimestamp: "2026-07-02T14:05:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:56Z" name: isvc-secondary-6f2d5c-predictor.18be7e1af5a27b4e namespace: kserve-ci-e2e-test resourceVersion: "20217" uid: 52de6aa8-552c-4faf-862f-83bb82e7a706 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-6f2d5c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20064" uid: 640d8c58-81a5-449e-97eb-a86f5289f5f8 kind: Event lastTimestamp: "2026-07-02T14:05:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:56Z" name: isvc-secondary-6f2d5c-predictor.18be7e1af615e143 namespace: kserve-ci-e2e-test resourceVersion: "20219" uid: 1e814d14-dd2a-4b2c-9b60-9fe80a5519b4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T14:05:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-6f2d5c namespace: kserve-ci-e2e-test resourceVersion: "20056" uid: c33409c7-729e-4872-ba8d-604f1807973a kind: Event lastTimestamp: "2026-07-02T14:05:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:05:55Z" name: isvc-secondary-6f2d5c.18be7e1777221fc3 namespace: kserve-ci-e2e-test resourceVersion: "20206" uid: 6e4e0704-7b17-4470-abb5-958771fea0da reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:05:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-6f2d5c namespace: kserve-ci-e2e-test resourceVersion: "20056" uid: c33409c7-729e-4872-ba8d-604f1807973a kind: Event lastTimestamp: "2026-07-02T14:05:41Z" message: 'Failed to update status for InferenceService "isvc-secondary-6f2d5c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-6f2d5c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:05:41Z" name: isvc-secondary-6f2d5c.18be7e1777983bda namespace: kserve-ci-e2e-test resourceVersion: "20084" uid: 0193a212-ff18-4f1a-8d71-5c47fdbb0f09 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:05:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-6f2d5c namespace: kserve-ci-e2e-test resourceVersion: "20056" uid: c33409c7-729e-4872-ba8d-604f1807973a kind: Event lastTimestamp: "2026-07-02T14:05:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-6f2d5c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:05:41Z" name: isvc-secondary-6f2d5c.18be7e1777987576 namespace: kserve-ci-e2e-test resourceVersion: "20085" uid: 69751581-e333-479f-be1f-c9c1c072ee9f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:55:40.164481Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: f67a4fdd-7894-447b-8301-28a285ee54e3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h to ip-10-0-136-2.ec2.internal metadata: creationTimestamp: "2026-07-02T13:55:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:55:40Z" name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h.18be7d8b66d579e6 namespace: kserve-ci-e2e-test resourceVersion: "14885" uid: 783d9d84-6535-454b-b92e-4737952e19b7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8899f5b4d-w9vws source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:40Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: f67a4fdd-7894-447b-8301-28a285ee54e3 kind: Event lastTimestamp: "2026-07-02T13:55:40Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:55:40Z" name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h.18be7d8b80d6a51d namespace: kserve-ci-e2e-test resourceVersion: "14921" uid: c7954f27-9df3-45e8-80f6-0bc1b84b05b8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h namespace: kserve-ci-e2e-test resourceVersion: "14874" uid: f67a4fdd-7894-447b-8301-28a285ee54e3 kind: Event lastTimestamp: "2026-07-02T13:55:40Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd2b86b0a2c9500aabe87a633c3b9bab30a880b19ff42859e99cc17f113ecdb0" already present on machine metadata: creationTimestamp: "2026-07-02T13:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:40Z" name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h.18be7d8b82beb95f namespace: kserve-ci-e2e-test resourceVersion: "14923" uid: 45a759a0-5e2e-41b8-8a92-f7d96f82042f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h namespace: kserve-ci-e2e-test resourceVersion: "14874" uid: f67a4fdd-7894-447b-8301-28a285ee54e3 kind: Event lastTimestamp: "2026-07-02T13:55:40Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T13:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:40Z" name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h.18be7d8b865b099b namespace: kserve-ci-e2e-test resourceVersion: "14927" uid: e4c77fd8-26a6-4d7a-a697-0ae6c8f08472 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h namespace: kserve-ci-e2e-test resourceVersion: "14874" uid: f67a4fdd-7894-447b-8301-28a285ee54e3 kind: Event lastTimestamp: "2026-07-02T13:55:40Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T13:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:40Z" name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h.18be7d8b86cb212b namespace: kserve-ci-e2e-test resourceVersion: "14928" uid: 35c3cc96-7147-4103-a6d9-f95f75cf9669 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h namespace: kserve-ci-e2e-test resourceVersion: "14874" uid: f67a4fdd-7894-447b-8301-28a285ee54e3 kind: Event lastTimestamp: "2026-07-02T13:55:46Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1695" already present on machine metadata: creationTimestamp: "2026-07-02T13:55:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:46Z" name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h.18be7d8cc725f5a7 namespace: kserve-ci-e2e-test resourceVersion: "14987" uid: cbb4ac8c-6392-4416-a391-04c44f89bdfb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h namespace: kserve-ci-e2e-test resourceVersion: "14874" uid: f67a4fdd-7894-447b-8301-28a285ee54e3 kind: Event lastTimestamp: "2026-07-02T13:55:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:55:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:46Z" name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h.18be7d8cf3e877ac namespace: kserve-ci-e2e-test resourceVersion: "14995" uid: effd7b83-b824-4797-8a6c-04afd8dd3d46 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h namespace: kserve-ci-e2e-test resourceVersion: "14874" uid: f67a4fdd-7894-447b-8301-28a285ee54e3 kind: Event lastTimestamp: "2026-07-02T13:55:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:55:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:46Z" name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h.18be7d8cf460735b namespace: kserve-ci-e2e-test resourceVersion: "14997" uid: 913419c2-40e4-479f-bb44-8f2aa2aef5d0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h namespace: kserve-ci-e2e-test resourceVersion: "14874" uid: f67a4fdd-7894-447b-8301-28a285ee54e3 kind: Event lastTimestamp: "2026-07-02T13:55:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:55:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:46Z" name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h.18be7d8cf46c2e0b namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: 3e459b88-8b5b-4dcd-ad94-5e75e53e1eb2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h namespace: kserve-ci-e2e-test resourceVersion: "14874" uid: f67a4fdd-7894-447b-8301-28a285ee54e3 kind: Event lastTimestamp: "2026-07-02T13:55:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:55:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:46Z" name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h.18be7d8cf7c21dc8 namespace: kserve-ci-e2e-test resourceVersion: "14999" uid: a16a3ff5-a7a4-4dc0-9c96-c5c4e6654fe4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h namespace: kserve-ci-e2e-test resourceVersion: "14874" uid: f67a4fdd-7894-447b-8301-28a285ee54e3 kind: Event lastTimestamp: "2026-07-02T13:55:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:55:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:46Z" name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h.18be7d8cf83e2560 namespace: kserve-ci-e2e-test resourceVersion: "15000" uid: fc29e9c0-7726-4341-9979-ab01310f11e2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T13:55:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h namespace: kserve-ci-e2e-test resourceVersion: "14874" uid: f67a4fdd-7894-447b-8301-28a285ee54e3 kind: Event lastTimestamp: "2026-07-02T13:57:34Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:34Z" name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h.18be7d8d582f4403 namespace: kserve-ci-e2e-test resourceVersion: "16064" uid: 67968ba6-99c2-44eb-99d1-a898e7a776df reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h namespace: kserve-ci-e2e-test resourceVersion: "14874" uid: f67a4fdd-7894-447b-8301-28a285ee54e3 kind: Event lastTimestamp: "2026-07-02T13:57:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:30Z" name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h.18be7da51fbba518 namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: 83b67dad-3c4d-4de7-b303-0f331a06892d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h namespace: kserve-ci-e2e-test resourceVersion: "14874" uid: f67a4fdd-7894-447b-8301-28a285ee54e3 kind: Event lastTimestamp: "2026-07-02T13:57:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:30Z" name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h.18be7da51fbceede namespace: kserve-ci-e2e-test resourceVersion: "15966" uid: 09c04748-1ce9-4ca8-8602-584afedb2aa8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h namespace: kserve-ci-e2e-test resourceVersion: "14874" uid: f67a4fdd-7894-447b-8301-28a285ee54e3 kind: Event lastTimestamp: "2026-07-02T13:57:34Z" message: 'Readiness probe failed: Get "https://10.133.0.28:8643/healthz": dial tcp 10.133.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:57:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:34Z" name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h.18be7da6068c1361 namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: 938477af-38f6-48eb-9a0d-a0e8d58655f7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc namespace: kserve-ci-e2e-test resourceVersion: "14839" uid: 1a2d3f55-a75d-4e52-89cc-decc8ddf9db6 kind: Event lastTimestamp: "2026-07-02T13:55:40Z" message: 'Created pod: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc-lhd4h' metadata: creationTimestamp: "2026-07-02T13:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:55:40Z" name: isvc-sklearn-graph-raw-8e592-predictor-557bc657fc.18be7d8b6613b0f0 namespace: kserve-ci-e2e-test resourceVersion: "14873" uid: 73fa1365-7cf2-4dc3-a158-1c10985a7968 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-8e592-predictor namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: fdcf1156-c82a-450d-86fb-ca276ad76854 kind: Event lastTimestamp: "2026-07-02T13:55:40Z" message: Scaled up replica set isvc-sklearn-graph-raw-8e592-predictor-557bc657fc from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:55:40Z" name: isvc-sklearn-graph-raw-8e592-predictor.18be7d8b60c442fb namespace: kserve-ci-e2e-test resourceVersion: "14840" uid: b4b73bf1-0ab3-4601-93f2-021ddd31d4d9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:55:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-8e592-predictor namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: 3b258a1a-6acf-4292-a66e-6925a235514f kind: Event lastTimestamp: "2026-07-02T13:56:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:55:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:56:25Z" name: isvc-sklearn-graph-raw-8e592-predictor.18be7d8edf7f6752 namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: 6591e0ff-820e-443a-b297-460f37f5cba6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:55:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-8e592-predictor namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: 3b258a1a-6acf-4292-a66e-6925a235514f kind: Event lastTimestamp: "2026-07-02T13:56:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:55:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:56:25Z" name: isvc-sklearn-graph-raw-8e592-predictor.18be7d8edfeaea76 namespace: kserve-ci-e2e-test resourceVersion: "15293" uid: 82c60d5b-bcc1-47a5-a796-4a0dbadfe505 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:56:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-8e592-predictor namespace: kserve-ci-e2e-test resourceVersion: "15058" uid: 3b258a1a-6acf-4292-a66e-6925a235514f kind: Event lastTimestamp: "2026-07-02T13:57:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:56:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:57:25Z" name: isvc-sklearn-graph-raw-8e592-predictor.18be7d995cf114a7 namespace: kserve-ci-e2e-test resourceVersion: "15814" uid: c6d2d98d-c7f3-4af8-884a-9f38c29ec2cc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:56:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-8e592-predictor namespace: kserve-ci-e2e-test resourceVersion: "15058" uid: 3b258a1a-6acf-4292-a66e-6925a235514f kind: Event lastTimestamp: "2026-07-02T13:57:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:56:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:57:25Z" name: isvc-sklearn-graph-raw-8e592-predictor.18be7d995d5ac948 namespace: kserve-ci-e2e-test resourceVersion: "15815" uid: 200ce598-6e8b-4e84-864d-20c11a3ef2a4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T13:55:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-8e592 namespace: kserve-ci-e2e-test resourceVersion: "14824" uid: 87a7b8a8-dd8b-440a-ab78-e2c0e3a873e5 kind: Event lastTimestamp: "2026-07-02T13:57:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:57:04Z" name: isvc-sklearn-graph-raw-8e592.18be7d8b60dbb755 namespace: kserve-ci-e2e-test resourceVersion: "15602" uid: 43b5a114-95cd-4031-8249-64036a7f9ad1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-8e592 namespace: kserve-ci-e2e-test resourceVersion: "14824" uid: 87a7b8a8-dd8b-440a-ab78-e2c0e3a873e5 kind: Event lastTimestamp: "2026-07-02T13:55:40Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-8e592": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-8e592": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:55:40Z" name: isvc-sklearn-graph-raw-8e592.18be7d8b6158bd8b namespace: kserve-ci-e2e-test resourceVersion: "14843" uid: 5707fa32-c35b-418e-aa69-3c77cb5f06c3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-8e592 namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: 87a7b8a8-dd8b-440a-ab78-e2c0e3a873e5 kind: Event lastTimestamp: "2026-07-02T13:57:04Z" message: InferenceService [isvc-sklearn-graph-raw-8e592] is Ready metadata: creationTimestamp: "2026-07-02T13:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:57:04Z" name: isvc-sklearn-graph-raw-8e592.18be7d9f124e3878 namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: bac31d59-77e5-4f01-b58b-dc919f8bdb44 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:57:30.617240Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv namespace: kserve-ci-e2e-test resourceVersion: "15935" uid: 230a6f67-46d5-417c-a60e-12e235c73863 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv to ip-10-0-136-2.ec2.internal metadata: creationTimestamp: "2026-07-02T13:57:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:57:30Z" name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv.18be7da51e54e1c2 namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: bd5d2047-7109-4f3d-9935-caf466afa986 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8899f5b4d-w9vws source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:31Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv namespace: kserve-ci-e2e-test resourceVersion: "15960" uid: 230a6f67-46d5-417c-a60e-12e235c73863 kind: Event lastTimestamp: "2026-07-02T13:57:31Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:57:31Z" name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv.18be7da53801dd5b namespace: kserve-ci-e2e-test resourceVersion: "16017" uid: c0cb8cb9-6774-45d6-ae80-798b68ff769d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv namespace: kserve-ci-e2e-test resourceVersion: "15940" uid: 230a6f67-46d5-417c-a60e-12e235c73863 kind: Event lastTimestamp: "2026-07-02T13:57:31Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd2b86b0a2c9500aabe87a633c3b9bab30a880b19ff42859e99cc17f113ecdb0" already present on machine metadata: creationTimestamp: "2026-07-02T13:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:31Z" name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv.18be7da53a4c7e58 namespace: kserve-ci-e2e-test resourceVersion: "16019" uid: cc7a296d-59b3-49a9-986a-f4787bd946e9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv namespace: kserve-ci-e2e-test resourceVersion: "15940" uid: 230a6f67-46d5-417c-a60e-12e235c73863 kind: Event lastTimestamp: "2026-07-02T13:57:31Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T13:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:31Z" name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv.18be7da53e05e033 namespace: kserve-ci-e2e-test resourceVersion: "16020" uid: c80bedca-8088-44a3-bd4f-9f0c60d55b90 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv namespace: kserve-ci-e2e-test resourceVersion: "15940" uid: 230a6f67-46d5-417c-a60e-12e235c73863 kind: Event lastTimestamp: "2026-07-02T13:57:31Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T13:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:31Z" name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv.18be7da53e723561 namespace: kserve-ci-e2e-test resourceVersion: "16022" uid: c6c026e1-6959-450a-9002-c63f04448859 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv namespace: kserve-ci-e2e-test resourceVersion: "15940" uid: 230a6f67-46d5-417c-a60e-12e235c73863 kind: Event lastTimestamp: "2026-07-02T13:57:35Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1695" already present on machine metadata: creationTimestamp: "2026-07-02T13:57:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:35Z" name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv.18be7da62586b0bf namespace: kserve-ci-e2e-test resourceVersion: "16071" uid: 9988e2de-ca27-417a-b8ce-234f423f60f8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv namespace: kserve-ci-e2e-test resourceVersion: "15940" uid: 230a6f67-46d5-417c-a60e-12e235c73863 kind: Event lastTimestamp: "2026-07-02T13:57:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:57:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:35Z" name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv.18be7da62a693ba0 namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: 4465f601-c82f-420a-8c65-7d53da08cdb8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv namespace: kserve-ci-e2e-test resourceVersion: "15940" uid: 230a6f67-46d5-417c-a60e-12e235c73863 kind: Event lastTimestamp: "2026-07-02T13:57:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:57:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:35Z" name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv.18be7da62ad75e1d namespace: kserve-ci-e2e-test resourceVersion: "16080" uid: a9a74fca-0b92-4850-8ae3-250cdb9fcd24 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv namespace: kserve-ci-e2e-test resourceVersion: "15940" uid: 230a6f67-46d5-417c-a60e-12e235c73863 kind: Event lastTimestamp: "2026-07-02T13:57:35Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:57:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:35Z" name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv.18be7da62ae1c3fe namespace: kserve-ci-e2e-test resourceVersion: "16081" uid: d6e0217a-a68e-43d7-93f2-7e118bec771d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv namespace: kserve-ci-e2e-test resourceVersion: "15940" uid: 230a6f67-46d5-417c-a60e-12e235c73863 kind: Event lastTimestamp: "2026-07-02T13:57:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:57:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:35Z" name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv.18be7da62e0df3b6 namespace: kserve-ci-e2e-test resourceVersion: "16086" uid: 976044f8-9606-4a67-8678-c95e519c3c9b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv namespace: kserve-ci-e2e-test resourceVersion: "15940" uid: 230a6f67-46d5-417c-a60e-12e235c73863 kind: Event lastTimestamp: "2026-07-02T13:57:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:57:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:35Z" name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv.18be7da62e74a8f1 namespace: kserve-ci-e2e-test resourceVersion: "16088" uid: 3b7b3b66-8b9d-49c4-8317-1ca39bc2ccd2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T13:57:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv namespace: kserve-ci-e2e-test resourceVersion: "15940" uid: 230a6f67-46d5-417c-a60e-12e235c73863 kind: Event lastTimestamp: "2026-07-02T13:59:23Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:23Z" name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv.18be7da69dc48c82 namespace: kserve-ci-e2e-test resourceVersion: "17050" uid: a94b29d6-34d6-422a-8c81-fcfab0e58afb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv namespace: kserve-ci-e2e-test resourceVersion: "15940" uid: 230a6f67-46d5-417c-a60e-12e235c73863 kind: Event lastTimestamp: "2026-07-02T13:59:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:59:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:20Z" name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv.18be7dbecf1bb353 namespace: kserve-ci-e2e-test resourceVersion: "16994" uid: f5e19079-37b7-474c-a1a5-259b621732ba reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv namespace: kserve-ci-e2e-test resourceVersion: "15940" uid: 230a6f67-46d5-417c-a60e-12e235c73863 kind: Event lastTimestamp: "2026-07-02T13:59:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:59:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:20Z" name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv.18be7dbecf1c1561 namespace: kserve-ci-e2e-test resourceVersion: "17000" uid: 507913d6-7761-4c20-a7d7-8c867ad9740d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv namespace: kserve-ci-e2e-test resourceVersion: "15940" uid: 230a6f67-46d5-417c-a60e-12e235c73863 kind: Event lastTimestamp: "2026-07-02T13:59:23Z" message: 'Readiness probe failed: Get "https://10.133.0.30:8643/healthz": dial tcp 10.133.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:59:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:23Z" name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv.18be7dbf4c1cba93 namespace: kserve-ci-e2e-test resourceVersion: "17048" uid: 16a50365-34a3-4e64-b9b1-cf6c422aeb1f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7 namespace: kserve-ci-e2e-test resourceVersion: "15916" uid: 197d73e0-2005-4c93-9f1c-21cb040a42d1 kind: Event lastTimestamp: "2026-07-02T13:57:30Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7-fg2sv' metadata: creationTimestamp: "2026-07-02T13:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:57:30Z" name: isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7.18be7da51dcf810d namespace: kserve-ci-e2e-test resourceVersion: "15944" uid: 7d638a37-c12b-46a4-b40b-4bcec1e5c763 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-e16db-predictor namespace: kserve-ci-e2e-test resourceVersion: "15913" uid: d0358abc-bc3e-4a4b-b6f0-d2fd38bb53fd kind: Event lastTimestamp: "2026-07-02T13:57:30Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-e16db-predictor-7b7f64c5c7 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:57:30Z" name: isvc-sklearn-graph-raw-hpa-e16db-predictor.18be7da51944c0a9 namespace: kserve-ci-e2e-test resourceVersion: "15917" uid: cfaad1cb-eef3-48b7-b0c3-534b97dc42ed reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:57:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-e16db-predictor namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: b1db7604-f2e5-47f6-83b2-0324a747ed9a kind: Event lastTimestamp: "2026-07-02T13:58:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:58:15Z" name: isvc-sklearn-graph-raw-hpa-e16db-predictor.18be7da8984109f6 namespace: kserve-ci-e2e-test resourceVersion: "16406" uid: 3536f0be-ada6-4e67-84b5-e9f127126324 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:57:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-e16db-predictor namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: b1db7604-f2e5-47f6-83b2-0324a747ed9a kind: Event lastTimestamp: "2026-07-02T13:58:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:58:15Z" name: isvc-sklearn-graph-raw-hpa-e16db-predictor.18be7da898abb4f5 namespace: kserve-ci-e2e-test resourceVersion: "16407" uid: 244b4113-acb2-404e-ba5f-7fe5dbf59252 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:58:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-e16db-predictor namespace: kserve-ci-e2e-test resourceVersion: "16177" uid: b1db7604-f2e5-47f6-83b2-0324a747ed9a kind: Event lastTimestamp: "2026-07-02T13:59:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:59:15Z" name: isvc-sklearn-graph-raw-hpa-e16db-predictor.18be7db3157e1cf4 namespace: kserve-ci-e2e-test resourceVersion: "16891" uid: 4baaba37-e308-4039-8b0c-a2618869736f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:58:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-e16db-predictor namespace: kserve-ci-e2e-test resourceVersion: "16177" uid: b1db7604-f2e5-47f6-83b2-0324a747ed9a kind: Event lastTimestamp: "2026-07-02T13:59:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:59:15Z" name: isvc-sklearn-graph-raw-hpa-e16db-predictor.18be7db315ed6c13 namespace: kserve-ci-e2e-test resourceVersion: "16892" uid: 5e426704-8bcd-4d72-8d73-f2c9c749d0d7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T13:57:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-e16db namespace: kserve-ci-e2e-test resourceVersion: "15905" uid: 36fab1f9-543c-4e2d-9893-00f8a340f2d6 kind: Event lastTimestamp: "2026-07-02T13:58:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:58:53Z" name: isvc-sklearn-graph-raw-hpa-e16db.18be7da5197eeefc namespace: kserve-ci-e2e-test resourceVersion: "16698" uid: 035f06c8-ed46-4486-a854-63c1b73aea1e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-e16db namespace: kserve-ci-e2e-test resourceVersion: "15905" uid: 36fab1f9-543c-4e2d-9893-00f8a340f2d6 kind: Event lastTimestamp: "2026-07-02T13:57:30Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-e16db": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-e16db": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:57:30Z" name: isvc-sklearn-graph-raw-hpa-e16db.18be7da519dc1e7a namespace: kserve-ci-e2e-test resourceVersion: "15920" uid: fede5792-370a-4086-bea2-34ef4fb9ba08 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:58:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-e16db namespace: kserve-ci-e2e-test resourceVersion: "16696" uid: 36fab1f9-543c-4e2d-9893-00f8a340f2d6 kind: Event lastTimestamp: "2026-07-02T13:58:53Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-e16db] is Ready metadata: creationTimestamp: "2026-07-02T13:58:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:58:53Z" name: isvc-sklearn-graph-raw-hpa-e16db.18be7db8575a8e9c namespace: kserve-ci-e2e-test resourceVersion: "16697" uid: 6c62f8fb-3687-40d7-8b59-50cac18b11cb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:01:16.267008Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2 namespace: kserve-ci-e2e-test resourceVersion: "18019" uid: 0dc540ec-5678-4ea3-8a6b-5f8ffc43ecd3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2 to ip-10-0-136-2.ec2.internal metadata: creationTimestamp: "2026-07-02T14:01:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:01:16Z" name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2.18be7dd9a81b0605 namespace: kserve-ci-e2e-test resourceVersion: "18028" uid: e938e196-8415-4b43-bccf-0f15cf9a3cc5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8899f5b4d-w9vws source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:01:16Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2 namespace: kserve-ci-e2e-test resourceVersion: "18041" uid: 0dc540ec-5678-4ea3-8a6b-5f8ffc43ecd3 kind: Event lastTimestamp: "2026-07-02T14:01:16Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:01:16Z" name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2.18be7dd9c1d70e45 namespace: kserve-ci-e2e-test resourceVersion: "18056" uid: 7786e7cb-1a61-4ded-94a4-73b4757e93a2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:01:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2 namespace: kserve-ci-e2e-test resourceVersion: "18022" uid: 0dc540ec-5678-4ea3-8a6b-5f8ffc43ecd3 kind: Event lastTimestamp: "2026-07-02T14:01:16Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd2b86b0a2c9500aabe87a633c3b9bab30a880b19ff42859e99cc17f113ecdb0" already present on machine metadata: creationTimestamp: "2026-07-02T14:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:01:16Z" name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2.18be7dd9c3539e97 namespace: kserve-ci-e2e-test resourceVersion: "18058" uid: bdd618dd-c4e0-42c0-b8bb-fc9b820227b3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:01:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2 namespace: kserve-ci-e2e-test resourceVersion: "18022" uid: 0dc540ec-5678-4ea3-8a6b-5f8ffc43ecd3 kind: Event lastTimestamp: "2026-07-02T14:01:16Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:01:16Z" name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2.18be7dd9c6428013 namespace: kserve-ci-e2e-test resourceVersion: "18059" uid: 2fd9726e-19c3-414f-b6e0-844982960739 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:01:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2 namespace: kserve-ci-e2e-test resourceVersion: "18022" uid: 0dc540ec-5678-4ea3-8a6b-5f8ffc43ecd3 kind: Event lastTimestamp: "2026-07-02T14:01:16Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:01:16Z" name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2.18be7dd9c6baa8ef namespace: kserve-ci-e2e-test resourceVersion: "18060" uid: 81bc15cf-951d-4858-83dd-587b12acbb81 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:01:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2 namespace: kserve-ci-e2e-test resourceVersion: "18022" uid: 0dc540ec-5678-4ea3-8a6b-5f8ffc43ecd3 kind: Event lastTimestamp: "2026-07-02T14:01:21Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1695" already present on machine metadata: creationTimestamp: "2026-07-02T14:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:01:21Z" name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2.18be7ddacb9f2d86 namespace: kserve-ci-e2e-test resourceVersion: "18098" uid: d8c829ef-77fe-4dc3-ac87-5ed39082561f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:01:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2 namespace: kserve-ci-e2e-test resourceVersion: "18022" uid: 0dc540ec-5678-4ea3-8a6b-5f8ffc43ecd3 kind: Event lastTimestamp: "2026-07-02T14:01:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:01:21Z" name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2.18be7ddaced2c473 namespace: kserve-ci-e2e-test resourceVersion: "18100" uid: c4ec5fb5-7285-4be3-9515-4aea02ec4227 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:01:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2 namespace: kserve-ci-e2e-test resourceVersion: "18022" uid: 0dc540ec-5678-4ea3-8a6b-5f8ffc43ecd3 kind: Event lastTimestamp: "2026-07-02T14:01:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:01:21Z" name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2.18be7ddacf45d5b1 namespace: kserve-ci-e2e-test resourceVersion: "18102" uid: 12eda39f-37f0-4905-a033-39786a8160cf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:01:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2 namespace: kserve-ci-e2e-test resourceVersion: "18022" uid: 0dc540ec-5678-4ea3-8a6b-5f8ffc43ecd3 kind: Event lastTimestamp: "2026-07-02T14:01:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:01:21Z" name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2.18be7ddacf50a59b namespace: kserve-ci-e2e-test resourceVersion: "18105" uid: 2bad3912-251f-4881-9caf-87bc115f120b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:01:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2 namespace: kserve-ci-e2e-test resourceVersion: "18022" uid: 0dc540ec-5678-4ea3-8a6b-5f8ffc43ecd3 kind: Event lastTimestamp: "2026-07-02T14:01:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:01:21Z" name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2.18be7ddad25de950 namespace: kserve-ci-e2e-test resourceVersion: "18107" uid: 39faf868-98ee-4106-bcdc-4eaa91455f7d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:01:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2 namespace: kserve-ci-e2e-test resourceVersion: "18022" uid: 0dc540ec-5678-4ea3-8a6b-5f8ffc43ecd3 kind: Event lastTimestamp: "2026-07-02T14:01:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:01:21Z" name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2.18be7ddad2d37f38 namespace: kserve-ci-e2e-test resourceVersion: "18108" uid: 2f02ae6f-7bd8-48d1-af8d-28c210a979c5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 19 eventTime: null firstTimestamp: "2026-07-02T14:01:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2 namespace: kserve-ci-e2e-test resourceVersion: "18022" uid: 0dc540ec-5678-4ea3-8a6b-5f8ffc43ecd3 kind: Event lastTimestamp: "2026-07-02T14:04:12Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:01:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:12Z" name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2.18be7ddb438db543 namespace: kserve-ci-e2e-test resourceVersion: "19393" uid: 3f96af91-5014-45be-9092-284cc52535c9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2 namespace: kserve-ci-e2e-test resourceVersion: "18022" uid: 0dc540ec-5678-4ea3-8a6b-5f8ffc43ecd3 kind: Event lastTimestamp: "2026-07-02T14:04:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:04:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:06Z" name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2.18be7e0147f31a1c namespace: kserve-ci-e2e-test resourceVersion: "19301" uid: bf654773-72c9-4b57-9373-1dada0971e55 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2 namespace: kserve-ci-e2e-test resourceVersion: "18022" uid: 0dc540ec-5678-4ea3-8a6b-5f8ffc43ecd3 kind: Event lastTimestamp: "2026-07-02T14:04:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:04:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:06Z" name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2.18be7e0147f401ef namespace: kserve-ci-e2e-test resourceVersion: "19302" uid: 37f54b7f-b3ab-4659-9c70-abfec70a52cf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:04:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2 namespace: kserve-ci-e2e-test resourceVersion: "18022" uid: 0dc540ec-5678-4ea3-8a6b-5f8ffc43ecd3 kind: Event lastTimestamp: "2026-07-02T14:04:14Z" message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": dial tcp 10.133.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:04:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:14Z" name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2.18be7e01ea2e94e5 namespace: kserve-ci-e2e-test resourceVersion: "19409" uid: 46e0a60a-be45-42ae-ab1e-5e5c916cc8b0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:01:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7 namespace: kserve-ci-e2e-test resourceVersion: "18002" uid: 503467fe-beeb-4d17-a78f-446912b96c64 kind: Event lastTimestamp: "2026-07-02T14:01:16Z" message: 'Created pod: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7-z4js2' metadata: creationTimestamp: "2026-07-02T14:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:01:16Z" name: isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7.18be7dd9a78303db namespace: kserve-ci-e2e-test resourceVersion: "18021" uid: 80bce46f-ace3-4b33-a0d7-a614bd03c615 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:01:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-1c082-predictor namespace: kserve-ci-e2e-test resourceVersion: "17994" uid: cc0a03a2-22ca-4585-9227-20f481190896 kind: Event lastTimestamp: "2026-07-02T14:01:16Z" message: Scaled up replica set isvc-sklearn-scale-raw-1c082-predictor-9f4ff4b7 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:01:16Z" name: isvc-sklearn-scale-raw-1c082-predictor.18be7dd9a2550f0d namespace: kserve-ci-e2e-test resourceVersion: "18003" uid: 97e415b5-c2e3-419a-852f-e1b9acbb7eaa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:01:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-1c082-predictor namespace: kserve-ci-e2e-test resourceVersion: "18001" uid: 37c8a521-7e8c-40fe-8b86-0531ae1b5d20 kind: Event lastTimestamp: "2026-07-02T14:02:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:01:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:02:01Z" name: isvc-sklearn-scale-raw-1c082-predictor.18be7ddd2103b83c namespace: kserve-ci-e2e-test resourceVersion: "18399" uid: d3f59b5c-b2ca-49ca-802b-4b2e67ed6658 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:01:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-1c082-predictor namespace: kserve-ci-e2e-test resourceVersion: "18001" uid: 37c8a521-7e8c-40fe-8b86-0531ae1b5d20 kind: Event lastTimestamp: "2026-07-02T14:02:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:01:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:02:01Z" name: isvc-sklearn-scale-raw-1c082-predictor.18be7ddd21670be6 namespace: kserve-ci-e2e-test resourceVersion: "18400" uid: 063b4f28-4713-410d-9925-9b876ed9d971 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-02T14:02:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-1c082-predictor namespace: kserve-ci-e2e-test resourceVersion: "18183" uid: 37c8a521-7e8c-40fe-8b86-0531ae1b5d20 kind: Event lastTimestamp: "2026-07-02T14:04:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:02:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:04:01Z" name: isvc-sklearn-scale-raw-1c082-predictor.18be7de79e9507da namespace: kserve-ci-e2e-test resourceVersion: "19241" uid: 2d35626d-42dd-494d-8ad8-5456e8ed07d3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-02T14:02:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-1c082-predictor namespace: kserve-ci-e2e-test resourceVersion: "18183" uid: 37c8a521-7e8c-40fe-8b86-0531ae1b5d20 kind: Event lastTimestamp: "2026-07-02T14:04:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:02:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:04:01Z" name: isvc-sklearn-scale-raw-1c082-predictor.18be7de79f042cbf namespace: kserve-ci-e2e-test resourceVersion: "19242" uid: 496f1b75-628c-47a1-8882-f54fba4af753 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T14:01:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-1c082 namespace: kserve-ci-e2e-test resourceVersion: "17980" uid: dcdc104f-9e4b-42b1-bf74-310fa8b1d7fe kind: Event lastTimestamp: "2026-07-02T14:04:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:04:02Z" name: isvc-sklearn-scale-raw-1c082.18be7dd9a2923316 namespace: kserve-ci-e2e-test resourceVersion: "19263" uid: 099ee24c-be63-4543-97d5-d5cca05e0402 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:01:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-1c082 namespace: kserve-ci-e2e-test resourceVersion: "17980" uid: dcdc104f-9e4b-42b1-bf74-310fa8b1d7fe kind: Event lastTimestamp: "2026-07-02T14:01:16Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-1c082": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-1c082": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:01:16Z" name: isvc-sklearn-scale-raw-1c082.18be7dd9a3320e79 namespace: kserve-ci-e2e-test resourceVersion: "18045" uid: 1732049d-c412-4241-b917-a32773647949 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:01:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-1c082 namespace: kserve-ci-e2e-test resourceVersion: "17997" uid: dcdc104f-9e4b-42b1-bf74-310fa8b1d7fe kind: Event lastTimestamp: "2026-07-02T14:01:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-1c082": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:01:16Z" name: isvc-sklearn-scale-raw-1c082.18be7dd9aa8acc66 namespace: kserve-ci-e2e-test resourceVersion: "18047" uid: f45709fb-a661-4c68-8feb-2db679382d11 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-1c082 namespace: kserve-ci-e2e-test resourceVersion: "19261" uid: dcdc104f-9e4b-42b1-bf74-310fa8b1d7fe kind: Event lastTimestamp: "2026-07-02T14:04:02Z" message: InferenceService [isvc-sklearn-scale-raw-1c082] is Ready metadata: creationTimestamp: "2026-07-02T14:04:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:04:02Z" name: isvc-sklearn-scale-raw-1c082.18be7e006478eacb namespace: kserve-ci-e2e-test resourceVersion: "19262" uid: 3af73c70-0d14-4542-aab2-4e06f7013782 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:55:40.240583Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw namespace: kserve-ci-e2e-test resourceVersion: "14896" uid: 4b7478df-d9a6-4eaa-8128-8dd3254fe5b3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw to ip-10-0-136-2.ec2.internal metadata: creationTimestamp: "2026-07-02T13:55:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:55:40Z" name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw.18be7d8b6b5ec351 namespace: kserve-ci-e2e-test resourceVersion: "14901" uid: a8d8612d-86ca-496c-a7b2-3a3e85dad38c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8899f5b4d-w9vws source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:40Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw namespace: kserve-ci-e2e-test resourceVersion: "14906" uid: 4b7478df-d9a6-4eaa-8128-8dd3254fe5b3 kind: Event lastTimestamp: "2026-07-02T13:55:40Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:55:40Z" name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw.18be7d8b8572bac4 namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 2805a010-3e84-4578-bbff-130e68e31927 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw namespace: kserve-ci-e2e-test resourceVersion: "14899" uid: 4b7478df-d9a6-4eaa-8128-8dd3254fe5b3 kind: Event lastTimestamp: "2026-07-02T13:55:40Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd2b86b0a2c9500aabe87a633c3b9bab30a880b19ff42859e99cc17f113ecdb0" already present on machine metadata: creationTimestamp: "2026-07-02T13:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:40Z" name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw.18be7d8b86eb2548 namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: 79051aec-56a0-45d7-8854-304833391055 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw namespace: kserve-ci-e2e-test resourceVersion: "14899" uid: 4b7478df-d9a6-4eaa-8128-8dd3254fe5b3 kind: Event lastTimestamp: "2026-07-02T13:55:40Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T13:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:40Z" name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw.18be7d8b89c10084 namespace: kserve-ci-e2e-test resourceVersion: "14930" uid: 3b1f99d3-3922-4f69-aacc-1322104ae44a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw namespace: kserve-ci-e2e-test resourceVersion: "14899" uid: 4b7478df-d9a6-4eaa-8128-8dd3254fe5b3 kind: Event lastTimestamp: "2026-07-02T13:55:40Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T13:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:40Z" name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw.18be7d8b8a22d36c namespace: kserve-ci-e2e-test resourceVersion: "14931" uid: 7effabaa-8b42-43ee-9b84-655cba4757d1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw namespace: kserve-ci-e2e-test resourceVersion: "14899" uid: 4b7478df-d9a6-4eaa-8128-8dd3254fe5b3 kind: Event lastTimestamp: "2026-07-02T13:55:44Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-02T13:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:44Z" name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw.18be7d8c67caa45c namespace: kserve-ci-e2e-test resourceVersion: "14969" uid: 8fe922c3-7143-4881-b90a-8da9f6d29895 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw namespace: kserve-ci-e2e-test resourceVersion: "14899" uid: 4b7478df-d9a6-4eaa-8128-8dd3254fe5b3 kind: Event lastTimestamp: "2026-07-02T13:56:03Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 19.441s (19.441s including waiting). Image size: 1305293048 bytes.' metadata: creationTimestamp: "2026-07-02T13:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:03Z" name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw.18be7d90ee95cdaf namespace: kserve-ci-e2e-test resourceVersion: "15127" uid: 0f6cc0aa-491f-4cdf-b450-cefc43ce25f8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw namespace: kserve-ci-e2e-test resourceVersion: "14899" uid: 4b7478df-d9a6-4eaa-8128-8dd3254fe5b3 kind: Event lastTimestamp: "2026-07-02T13:56:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:03Z" name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw.18be7d90f1dc6c0a namespace: kserve-ci-e2e-test resourceVersion: "15128" uid: dee6ae18-1d13-4933-abaf-777e8b1fd88c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw namespace: kserve-ci-e2e-test resourceVersion: "14899" uid: 4b7478df-d9a6-4eaa-8128-8dd3254fe5b3 kind: Event lastTimestamp: "2026-07-02T13:56:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:03Z" name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw.18be7d90f24a3848 namespace: kserve-ci-e2e-test resourceVersion: "15129" uid: 286f08fc-4ca6-4863-8a43-04c4b37e6fe4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw namespace: kserve-ci-e2e-test resourceVersion: "14899" uid: 4b7478df-d9a6-4eaa-8128-8dd3254fe5b3 kind: Event lastTimestamp: "2026-07-02T13:56:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:03Z" name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw.18be7d90f2546fd1 namespace: kserve-ci-e2e-test resourceVersion: "15130" uid: ca6a68f1-a9b7-4bbc-b23f-a1cfd9ace064 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw namespace: kserve-ci-e2e-test resourceVersion: "14899" uid: 4b7478df-d9a6-4eaa-8128-8dd3254fe5b3 kind: Event lastTimestamp: "2026-07-02T13:56:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:04Z" name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw.18be7d90f57f4b03 namespace: kserve-ci-e2e-test resourceVersion: "15132" uid: a5ef5ac2-03f5-46c3-a6b4-79f88d0f780a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw namespace: kserve-ci-e2e-test resourceVersion: "14899" uid: 4b7478df-d9a6-4eaa-8128-8dd3254fe5b3 kind: Event lastTimestamp: "2026-07-02T13:56:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:04Z" name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw.18be7d90f5f1e1cb namespace: kserve-ci-e2e-test resourceVersion: "15133" uid: 03c13b9a-cec4-4406-8745-96f41a8546c3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-02T13:56:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw namespace: kserve-ci-e2e-test resourceVersion: "14899" uid: 4b7478df-d9a6-4eaa-8128-8dd3254fe5b3 kind: Event lastTimestamp: "2026-07-02T13:57:31Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:31Z" name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw.18be7d9152bc7c12 namespace: kserve-ci-e2e-test resourceVersion: "16030" uid: 6cb6ddf1-2bc7-4543-abb6-20378b9fded8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw namespace: kserve-ci-e2e-test resourceVersion: "14899" uid: 4b7478df-d9a6-4eaa-8128-8dd3254fe5b3 kind: Event lastTimestamp: "2026-07-02T13:57:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:30Z" name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw.18be7da527c0ab9a namespace: kserve-ci-e2e-test resourceVersion: "15999" uid: 6c416407-d74e-496d-a659-467181aca4e8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw namespace: kserve-ci-e2e-test resourceVersion: "14899" uid: 4b7478df-d9a6-4eaa-8128-8dd3254fe5b3 kind: Event lastTimestamp: "2026-07-02T13:57:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:30Z" name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw.18be7da527c110bf namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: 82c56998-a2ea-43b7-a81a-eaddbc95bc0d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw namespace: kserve-ci-e2e-test resourceVersion: "14899" uid: 4b7478df-d9a6-4eaa-8128-8dd3254fe5b3 kind: Event lastTimestamp: "2026-07-02T13:57:31Z" message: 'Readiness probe failed: Get "https://10.133.0.29:8643/healthz": dial tcp 10.133.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:31Z" name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw.18be7da558fe9476 namespace: kserve-ci-e2e-test resourceVersion: "16029" uid: b2ba7a27-1e5d-4915-9648-5744621ae7e1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689 namespace: kserve-ci-e2e-test resourceVersion: "14878" uid: bd9f9d0a-fd1a-4971-a0b6-2dfbefb963f9 kind: Event lastTimestamp: "2026-07-02T13:55:40Z" message: 'Created pod: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689-bs9jw' metadata: creationTimestamp: "2026-07-02T13:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:55:40Z" name: isvc-xgboost-graph-raw-8e592-predictor-75b67b8689.18be7d8b6ae693f5 namespace: kserve-ci-e2e-test resourceVersion: "14898" uid: e6a21ac9-43cc-4516-87ae-7b9f05461e74 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-8e592-predictor namespace: kserve-ci-e2e-test resourceVersion: "14872" uid: f276b03a-feb2-4f75-ba6d-c08d4e34ed65 kind: Event lastTimestamp: "2026-07-02T13:55:40Z" message: Scaled up replica set isvc-xgboost-graph-raw-8e592-predictor-75b67b8689 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:55:40Z" name: isvc-xgboost-graph-raw-8e592-predictor.18be7d8b6738d5af namespace: kserve-ci-e2e-test resourceVersion: "14883" uid: 774b89c1-dded-468b-ba6f-4d5a0d0472d1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:55:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-8e592-predictor namespace: kserve-ci-e2e-test resourceVersion: "14877" uid: aef81c71-19eb-4a8c-a97f-2e35548f0e42 kind: Event lastTimestamp: "2026-07-02T13:56:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:55:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:56:40Z" name: isvc-xgboost-graph-raw-8e592-predictor.18be7d8ee5bb6702 namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 5a3919b1-7066-45d8-907b-b56365c75356 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:55:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-8e592-predictor namespace: kserve-ci-e2e-test resourceVersion: "14877" uid: aef81c71-19eb-4a8c-a97f-2e35548f0e42 kind: Event lastTimestamp: "2026-07-02T13:56:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:55:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:56:40Z" name: isvc-xgboost-graph-raw-8e592-predictor.18be7d8ee63147e3 namespace: kserve-ci-e2e-test resourceVersion: "15411" uid: 316f5850-0299-432d-9413-d515d3cf240e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:56:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-8e592-predictor namespace: kserve-ci-e2e-test resourceVersion: "15062" uid: aef81c71-19eb-4a8c-a97f-2e35548f0e42 kind: Event lastTimestamp: "2026-07-02T13:57:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:56:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:57:25Z" name: isvc-xgboost-graph-raw-8e592-predictor.18be7d95e394e2b1 namespace: kserve-ci-e2e-test resourceVersion: "15817" uid: 238ab57b-0a38-4132-be09-98dc5816e901 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:56:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-8e592-predictor namespace: kserve-ci-e2e-test resourceVersion: "15062" uid: aef81c71-19eb-4a8c-a97f-2e35548f0e42 kind: Event lastTimestamp: "2026-07-02T13:57:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:56:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:57:25Z" name: isvc-xgboost-graph-raw-8e592-predictor.18be7d95e3ffbd32 namespace: kserve-ci-e2e-test resourceVersion: "15818" uid: 32e0154b-fd54-4d39-a511-24574e2e5afd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T13:55:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-8e592 namespace: kserve-ci-e2e-test resourceVersion: "14846" uid: 78c52100-2bbb-4051-b730-4e549a8ebaf2 kind: Event lastTimestamp: "2026-07-02T13:57:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:57:01Z" name: isvc-xgboost-graph-raw-8e592.18be7d8b676ea282 namespace: kserve-ci-e2e-test resourceVersion: "15572" uid: 1b62b3cf-04fe-4ccd-ba8c-bd1027c7a781 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-8e592 namespace: kserve-ci-e2e-test resourceVersion: "14846" uid: 78c52100-2bbb-4051-b730-4e549a8ebaf2 kind: Event lastTimestamp: "2026-07-02T13:55:40Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-8e592": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-8e592": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:55:40Z" name: isvc-xgboost-graph-raw-8e592.18be7d8b67e3f884 namespace: kserve-ci-e2e-test resourceVersion: "14890" uid: 55a799a7-122e-418c-9e0d-b2792bec46aa reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-8e592 namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: 78c52100-2bbb-4051-b730-4e549a8ebaf2 kind: Event lastTimestamp: "2026-07-02T13:57:01Z" message: InferenceService [isvc-xgboost-graph-raw-8e592] is Ready metadata: creationTimestamp: "2026-07-02T13:57:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:57:01Z" name: isvc-xgboost-graph-raw-8e592.18be7d9e636501e1 namespace: kserve-ci-e2e-test resourceVersion: "15570" uid: 18d68092-f4a1-4d62-a379-dc0d4b8a1d9c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:57:30.718372Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w namespace: kserve-ci-e2e-test resourceVersion: "15971" uid: 2c619fa6-4ad0-4ab9-8aea-338438880b09 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w to ip-10-0-136-2.ec2.internal metadata: creationTimestamp: "2026-07-02T13:57:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:57:30Z" name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w.18be7da5245c2b4f namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: 800a103c-515d-4d1d-af2e-a19549fe0b95 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8899f5b4d-w9vws source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:31Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 2c619fa6-4ad0-4ab9-8aea-338438880b09 kind: Event lastTimestamp: "2026-07-02T13:57:31Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:57:31Z" name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w.18be7da53e295013 namespace: kserve-ci-e2e-test resourceVersion: "16021" uid: 0f648abf-1864-4fee-8439-b183a54aa143 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 2c619fa6-4ad0-4ab9-8aea-338438880b09 kind: Event lastTimestamp: "2026-07-02T13:57:31Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd2b86b0a2c9500aabe87a633c3b9bab30a880b19ff42859e99cc17f113ecdb0" already present on machine metadata: creationTimestamp: "2026-07-02T13:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:31Z" name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w.18be7da540014f4a namespace: kserve-ci-e2e-test resourceVersion: "16024" uid: 8e6bdb1b-e626-435b-b446-ce49e3cda63b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 2c619fa6-4ad0-4ab9-8aea-338438880b09 kind: Event lastTimestamp: "2026-07-02T13:57:31Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T13:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:31Z" name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w.18be7da543d6b73f namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 01acf16a-c5e9-4361-abcd-0565427f0699 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 2c619fa6-4ad0-4ab9-8aea-338438880b09 kind: Event lastTimestamp: "2026-07-02T13:57:31Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T13:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:31Z" name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w.18be7da544836011 namespace: kserve-ci-e2e-test resourceVersion: "16027" uid: 5dc2de01-4a49-48ed-8b81-1488c12bf549 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 2c619fa6-4ad0-4ab9-8aea-338438880b09 kind: Event lastTimestamp: "2026-07-02T13:57:35Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-07-02T13:57:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:35Z" name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w.18be7da6259e0311 namespace: kserve-ci-e2e-test resourceVersion: "16072" uid: fea2cdd3-6a55-49f5-b7fe-3a5e4aea5b03 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 2c619fa6-4ad0-4ab9-8aea-338438880b09 kind: Event lastTimestamp: "2026-07-02T13:57:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:57:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:35Z" name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w.18be7da62a881fd5 namespace: kserve-ci-e2e-test resourceVersion: "16079" uid: e4ad667f-287d-485f-adff-e954979b2de3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 2c619fa6-4ad0-4ab9-8aea-338438880b09 kind: Event lastTimestamp: "2026-07-02T13:57:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:57:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:35Z" name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w.18be7da62af43e14 namespace: kserve-ci-e2e-test resourceVersion: "16082" uid: 8f45ff19-448d-488e-ba06-c2387700c6b6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 2c619fa6-4ad0-4ab9-8aea-338438880b09 kind: Event lastTimestamp: "2026-07-02T13:57:35Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:57:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:35Z" name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w.18be7da62afdc223 namespace: kserve-ci-e2e-test resourceVersion: "16083" uid: 15a61394-a6db-4197-8f50-d73a9df0a4d8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 2c619fa6-4ad0-4ab9-8aea-338438880b09 kind: Event lastTimestamp: "2026-07-02T13:57:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:57:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:35Z" name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w.18be7da62e300969 namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: 188672ce-4ca7-41b4-bc9a-ac5ff0d4cb5a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 2c619fa6-4ad0-4ab9-8aea-338438880b09 kind: Event lastTimestamp: "2026-07-02T13:57:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:57:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:35Z" name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w.18be7da62e92af10 namespace: kserve-ci-e2e-test resourceVersion: "16089" uid: 245c8517-d6fa-4895-a4db-b1f4f4d34671 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-02T13:57:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 2c619fa6-4ad0-4ab9-8aea-338438880b09 kind: Event lastTimestamp: "2026-07-02T13:59:23Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:23Z" name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w.18be7da69dc473c1 namespace: kserve-ci-e2e-test resourceVersion: "17049" uid: 6bb3e6eb-ce28-4207-b55b-fd4d00f4824e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 2c619fa6-4ad0-4ab9-8aea-338438880b09 kind: Event lastTimestamp: "2026-07-02T13:59:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:59:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:21Z" name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w.18be7dbed1c35a53 namespace: kserve-ci-e2e-test resourceVersion: "17013" uid: fed173b8-3899-4828-9b14-efc619287a84 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 2c619fa6-4ad0-4ab9-8aea-338438880b09 kind: Event lastTimestamp: "2026-07-02T13:59:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:59:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:21Z" name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w.18be7dbed1c3bd35 namespace: kserve-ci-e2e-test resourceVersion: "17016" uid: 9d5891f1-1703-4cce-b7ae-cc0876f5964e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 2c619fa6-4ad0-4ab9-8aea-338438880b09 kind: Event lastTimestamp: "2026-07-02T13:59:23Z" message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp 10.133.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:59:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:23Z" name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w.18be7dbf4c1ba3b6 namespace: kserve-ci-e2e-test resourceVersion: "17047" uid: 2a64bea4-351a-47ef-9ea8-afafedb7111f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f namespace: kserve-ci-e2e-test resourceVersion: "15959" uid: 5494d3bb-0dab-4a63-9385-97f720f82ffb kind: Event lastTimestamp: "2026-07-02T13:57:30Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f-6fr2w' metadata: creationTimestamp: "2026-07-02T13:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:57:30Z" name: isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f.18be7da523a905d4 namespace: kserve-ci-e2e-test resourceVersion: "15972" uid: dba21192-2784-4589-b97d-0bf840565893 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-e16db-predictor namespace: kserve-ci-e2e-test resourceVersion: "15954" uid: 6340da71-c12e-423b-bc9f-c3181552b17b kind: Event lastTimestamp: "2026-07-02T13:57:30Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-e16db-predictor-666d58894f from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:57:30Z" name: isvc-xgboost-graph-raw-hpa-e16db-predictor.18be7da51fee6ec2 namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: c1b73f45-d8d7-47ee-bd81-c15ffbd613c2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:57:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-e16db-predictor namespace: kserve-ci-e2e-test resourceVersion: "15957" uid: 1320f174-912a-4bb0-91b6-a50e9eb5e809 kind: Event lastTimestamp: "2026-07-02T13:58:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:58:15Z" name: isvc-xgboost-graph-raw-hpa-e16db-predictor.18be7da89e6acd96 namespace: kserve-ci-e2e-test resourceVersion: "16409" uid: dd0edf3b-bf16-4935-8d65-48d31e83c246 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:57:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-e16db-predictor namespace: kserve-ci-e2e-test resourceVersion: "15957" uid: 1320f174-912a-4bb0-91b6-a50e9eb5e809 kind: Event lastTimestamp: "2026-07-02T13:58:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:58:15Z" name: isvc-xgboost-graph-raw-hpa-e16db-predictor.18be7da89ef5265b namespace: kserve-ci-e2e-test resourceVersion: "16410" uid: 4d702bc7-1eb6-42ad-b0d7-f8ff8b2394b8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:58:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-e16db-predictor namespace: kserve-ci-e2e-test resourceVersion: "16181" uid: 1320f174-912a-4bb0-91b6-a50e9eb5e809 kind: Event lastTimestamp: "2026-07-02T13:59:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:59:15Z" name: isvc-xgboost-graph-raw-hpa-e16db-predictor.18be7db31b298a08 namespace: kserve-ci-e2e-test resourceVersion: "16893" uid: 72e170d6-45f6-4c9a-a343-e7b6d120b94e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:58:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-e16db-predictor namespace: kserve-ci-e2e-test resourceVersion: "16181" uid: 1320f174-912a-4bb0-91b6-a50e9eb5e809 kind: Event lastTimestamp: "2026-07-02T13:59:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:59:15Z" name: isvc-xgboost-graph-raw-hpa-e16db-predictor.18be7db31b974438 namespace: kserve-ci-e2e-test resourceVersion: "16894" uid: ca7dac52-400e-45c3-bfda-9f60ffd3d2aa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T13:57:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-e16db namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: 0f3ba51d-8541-4b67-95ad-22e9fa7bec2d kind: Event lastTimestamp: "2026-07-02T13:58:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:58:33Z" name: isvc-xgboost-graph-raw-hpa-e16db.18be7da52015fb20 namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: 10ba52c0-d8ae-4856-ae65-770c5d0efa85 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-e16db namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: 0f3ba51d-8541-4b67-95ad-22e9fa7bec2d kind: Event lastTimestamp: "2026-07-02T13:57:30Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-hpa-e16db": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-hpa-e16db": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:57:30Z" name: isvc-xgboost-graph-raw-hpa-e16db.18be7da52119e374 namespace: kserve-ci-e2e-test resourceVersion: "15969" uid: 21fd937e-4ffc-4556-8d6b-c4f0a70a7c41 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:58:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-e16db namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 0f3ba51d-8541-4b67-95ad-22e9fa7bec2d kind: Event lastTimestamp: "2026-07-02T13:58:33Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-e16db] is Ready metadata: creationTimestamp: "2026-07-02T13:58:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:58:33Z" name: isvc-xgboost-graph-raw-hpa-e16db.18be7db3ae554a8e namespace: kserve-ci-e2e-test resourceVersion: "16549" uid: a4f3171e-e1eb-4563-b4b5-140c7379e20f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:59:20.903102Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-1b7e6-predictor-64df5c9c95-62h4v namespace: kserve-ci-e2e-test resourceVersion: "16967" uid: ee0558b3-ff74-4543-bd36-79ba2e659422 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-1b7e6-predictor-64df5c9c95-62h4v to ip-10-0-136-2.ec2.internal metadata: creationTimestamp: "2026-07-02T13:59:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:59:20Z" name: message-dumper-raw-1b7e6-predictor-64df5c9c95-62h4v.18be7dbecbe19ee0 namespace: kserve-ci-e2e-test resourceVersion: "16977" uid: 293625f1-a77d-4d8a-aa29-f49ea7497ac1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8899f5b4d-w9vws source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:21Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-1b7e6-predictor-64df5c9c95-62h4v namespace: kserve-ci-e2e-test resourceVersion: "16971" uid: ee0558b3-ff74-4543-bd36-79ba2e659422 kind: Event lastTimestamp: "2026-07-02T13:59:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "message-dumper-raw-1b7e6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:59:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:21Z" name: message-dumper-raw-1b7e6-predictor-64df5c9c95-62h4v.18be7dbed8710773 namespace: kserve-ci-e2e-test resourceVersion: "17023" uid: 199fc22c-02da-4174-a80c-986ce734c41d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:21Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-1b7e6-predictor-64df5c9c95-62h4v namespace: kserve-ci-e2e-test resourceVersion: "16991" uid: ee0558b3-ff74-4543-bd36-79ba2e659422 kind: Event lastTimestamp: "2026-07-02T13:59:21Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:59:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:59:21Z" name: message-dumper-raw-1b7e6-predictor-64df5c9c95-62h4v.18be7dbf0945b4e8 namespace: kserve-ci-e2e-test resourceVersion: "17033" uid: 3136101d-e930-4f88-b689-c7264217e20d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-1b7e6-predictor-64df5c9c95-62h4v namespace: kserve-ci-e2e-test resourceVersion: "16971" uid: ee0558b3-ff74-4543-bd36-79ba2e659422 kind: Event lastTimestamp: "2026-07-02T13:59:21Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-02T13:59:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:21Z" name: message-dumper-raw-1b7e6-predictor-64df5c9c95-62h4v.18be7dbf0ab2e6f1 namespace: kserve-ci-e2e-test resourceVersion: "17035" uid: 03f5f585-d283-404b-bd5e-43fa506358b8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-1b7e6-predictor-64df5c9c95-62h4v namespace: kserve-ci-e2e-test resourceVersion: "16971" uid: ee0558b3-ff74-4543-bd36-79ba2e659422 kind: Event lastTimestamp: "2026-07-02T13:59:23Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.204s (1.204s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-02T13:59:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:23Z" name: message-dumper-raw-1b7e6-predictor-64df5c9c95-62h4v.18be7dbf52797192 namespace: kserve-ci-e2e-test resourceVersion: "17051" uid: 3a99e257-6b7a-46bc-94c6-a80eb35ebdca reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-1b7e6-predictor-64df5c9c95-62h4v namespace: kserve-ci-e2e-test resourceVersion: "16971" uid: ee0558b3-ff74-4543-bd36-79ba2e659422 kind: Event lastTimestamp: "2026-07-02T13:59:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:59:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:23Z" name: message-dumper-raw-1b7e6-predictor-64df5c9c95-62h4v.18be7dbf55637c09 namespace: kserve-ci-e2e-test resourceVersion: "17053" uid: ac3c5f57-a11a-4c78-bd5a-8b59b6cbdcae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-1b7e6-predictor-64df5c9c95-62h4v namespace: kserve-ci-e2e-test resourceVersion: "16971" uid: ee0558b3-ff74-4543-bd36-79ba2e659422 kind: Event lastTimestamp: "2026-07-02T13:59:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:59:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:23Z" name: message-dumper-raw-1b7e6-predictor-64df5c9c95-62h4v.18be7dbf55c1c706 namespace: kserve-ci-e2e-test resourceVersion: "17054" uid: ee3cb589-7ccf-4927-987d-b6200a7b0ccd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-1b7e6-predictor-64df5c9c95-62h4v namespace: kserve-ci-e2e-test resourceVersion: "16971" uid: ee0558b3-ff74-4543-bd36-79ba2e659422 kind: Event lastTimestamp: "2026-07-02T13:59:23Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:59:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:23Z" name: message-dumper-raw-1b7e6-predictor-64df5c9c95-62h4v.18be7dbf55ca98d1 namespace: kserve-ci-e2e-test resourceVersion: "17055" uid: 3eeaf89d-fdaf-4e23-85be-ed0627ce6cb1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-1b7e6-predictor-64df5c9c95-62h4v namespace: kserve-ci-e2e-test resourceVersion: "16971" uid: ee0558b3-ff74-4543-bd36-79ba2e659422 kind: Event lastTimestamp: "2026-07-02T13:59:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:59:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:23Z" name: message-dumper-raw-1b7e6-predictor-64df5c9c95-62h4v.18be7dbf58a70274 namespace: kserve-ci-e2e-test resourceVersion: "17057" uid: 48dd346c-3a59-4f42-9a21-63d8f4416399 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-1b7e6-predictor-64df5c9c95-62h4v namespace: kserve-ci-e2e-test resourceVersion: "16971" uid: ee0558b3-ff74-4543-bd36-79ba2e659422 kind: Event lastTimestamp: "2026-07-02T13:59:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:59:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:23Z" name: message-dumper-raw-1b7e6-predictor-64df5c9c95-62h4v.18be7dbf590d661c namespace: kserve-ci-e2e-test resourceVersion: "17058" uid: a016f04e-c20c-4ebc-be66-cd474ab26c01 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:01:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-1b7e6-predictor-64df5c9c95-62h4v namespace: kserve-ci-e2e-test resourceVersion: "16971" uid: ee0558b3-ff74-4543-bd36-79ba2e659422 kind: Event lastTimestamp: "2026-07-02T14:01:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:01:16Z" name: message-dumper-raw-1b7e6-predictor-64df5c9c95-62h4v.18be7dd9aa873cd6 namespace: kserve-ci-e2e-test resourceVersion: "18043" uid: 42758fca-d772-41ba-9732-d45cdc1af348 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:01:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-1b7e6-predictor-64df5c9c95-62h4v namespace: kserve-ci-e2e-test resourceVersion: "16971" uid: ee0558b3-ff74-4543-bd36-79ba2e659422 kind: Event lastTimestamp: "2026-07-02T14:01:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:01:16Z" name: message-dumper-raw-1b7e6-predictor-64df5c9c95-62h4v.18be7dd9aa8e5fb5 namespace: kserve-ci-e2e-test resourceVersion: "18044" uid: 7b09fcc5-0c14-4dfc-a9b0-9102d549893d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-1b7e6-predictor-64df5c9c95 namespace: kserve-ci-e2e-test resourceVersion: "16958" uid: ae383f99-de73-4d6c-b1c7-2bf615c5331c kind: Event lastTimestamp: "2026-07-02T13:59:20Z" message: 'Created pod: message-dumper-raw-1b7e6-predictor-64df5c9c95-62h4v' metadata: creationTimestamp: "2026-07-02T13:59:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:59:20Z" name: message-dumper-raw-1b7e6-predictor-64df5c9c95.18be7dbecb5f3d23 namespace: kserve-ci-e2e-test resourceVersion: "16972" uid: d3089c89-c4a7-448b-80be-012ea186becc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-1b7e6-predictor namespace: kserve-ci-e2e-test resourceVersion: "16955" uid: 07b20e4d-dd20-49ba-8b3c-74e67c899fa1 kind: Event lastTimestamp: "2026-07-02T13:59:20Z" message: Scaled up replica set message-dumper-raw-1b7e6-predictor-64df5c9c95 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:59:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:59:20Z" name: message-dumper-raw-1b7e6-predictor.18be7dbec7d36ca9 namespace: kserve-ci-e2e-test resourceVersion: "16959" uid: 79a9076a-b8cc-4760-a2fc-834c445008b8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:59:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-1b7e6-predictor namespace: kserve-ci-e2e-test resourceVersion: "16957" uid: b89e07ae-5d65-45c0-890a-e74a7696395b kind: Event lastTimestamp: "2026-07-02T13:59:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:59:50Z" name: message-dumper-raw-1b7e6-predictor.18be7dc2465d73f8 namespace: kserve-ci-e2e-test resourceVersion: "17353" uid: e4820cdd-1907-4359-84d7-8e3bbf75bec4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:59:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-1b7e6-predictor namespace: kserve-ci-e2e-test resourceVersion: "16957" uid: b89e07ae-5d65-45c0-890a-e74a7696395b kind: Event lastTimestamp: "2026-07-02T13:59:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:59:50Z" name: message-dumper-raw-1b7e6-predictor.18be7dc246c6568e namespace: kserve-ci-e2e-test resourceVersion: "17354" uid: 57cc572f-509e-48df-92e6-5488a7d36926 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T13:59:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-1b7e6 namespace: kserve-ci-e2e-test resourceVersion: "16947" uid: 94191f9f-6e00-4505-8b98-54efa33975f0 kind: Event lastTimestamp: "2026-07-02T13:59:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:59:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:59:31Z" name: message-dumper-raw-1b7e6.18be7dbec7e00937 namespace: kserve-ci-e2e-test resourceVersion: "17136" uid: 291cebb2-cb5f-41bd-9af3-371991e24b95 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-1b7e6 namespace: kserve-ci-e2e-test resourceVersion: "16947" uid: 94191f9f-6e00-4505-8b98-54efa33975f0 kind: Event lastTimestamp: "2026-07-02T13:59:20Z" message: 'Failed to update status for InferenceService "message-dumper-raw-1b7e6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-1b7e6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:59:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:59:20Z" name: message-dumper-raw-1b7e6.18be7dbec840668f namespace: kserve-ci-e2e-test resourceVersion: "16962" uid: 9e95dc55-bdc8-4c65-9c7d-cd2543f3eb72 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-1b7e6 namespace: kserve-ci-e2e-test resourceVersion: "17133" uid: 94191f9f-6e00-4505-8b98-54efa33975f0 kind: Event lastTimestamp: "2026-07-02T13:59:31Z" message: InferenceService [message-dumper-raw-1b7e6] is Ready metadata: creationTimestamp: "2026-07-02T13:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:59:31Z" name: message-dumper-raw-1b7e6.18be7dc14e98499e namespace: kserve-ci-e2e-test resourceVersion: "17135" uid: ff0905b0-f53d-4f02-83ce-90c2cd553a4e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:57:20.190643Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-8e592-59dcb55855-b68v9 namespace: kserve-ci-e2e-test resourceVersion: "15755" uid: 65330a8b-788f-4862-86f8-008592863d58 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-8e592-59dcb55855-b68v9 to ip-10-0-140-174.ec2.internal metadata: creationTimestamp: "2026-07-02T13:57:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:57:20Z" name: model-chainer-raw-8e592-59dcb55855-b68v9.18be7da2b0dbab9d namespace: kserve-ci-e2e-test resourceVersion: "15760" uid: f33d755f-9c4b-4e35-9827-df4fab7df8c1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8899f5b4d-w9vws source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:20Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-8e592-59dcb55855-b68v9 namespace: kserve-ci-e2e-test resourceVersion: "15764" uid: 65330a8b-788f-4862-86f8-008592863d58 kind: Event lastTimestamp: "2026-07-02T13:57:20Z" message: Add eth0 [10.132.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:57:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:57:20Z" name: model-chainer-raw-8e592-59dcb55855-b68v9.18be7da2c9c5d6a3 namespace: kserve-ci-e2e-test resourceVersion: "15776" uid: 75b6273f-26d3-4aa4-b44a-ff412ba188a1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-8e592} kind: Pod name: model-chainer-raw-8e592-59dcb55855-b68v9 namespace: kserve-ci-e2e-test resourceVersion: "15759" uid: 65330a8b-788f-4862-86f8-008592863d58 kind: Event lastTimestamp: "2026-07-02T13:57:20Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:b87693b966d6607544232e73341be0a7df8bc57efa51891457e864fb9d5cc954" metadata: creationTimestamp: "2026-07-02T13:57:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:20Z" name: model-chainer-raw-8e592-59dcb55855-b68v9.18be7da2cb2d9321 namespace: kserve-ci-e2e-test resourceVersion: "15778" uid: 5c3dae44-aee7-4605-a9bb-9c422431cbea reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-174.ec2.internal source: component: kubelet host: ip-10-0-140-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-8e592} kind: Pod name: model-chainer-raw-8e592-59dcb55855-b68v9 namespace: kserve-ci-e2e-test resourceVersion: "15759" uid: 65330a8b-788f-4862-86f8-008592863d58 kind: Event lastTimestamp: "2026-07-02T13:57:23Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:b87693b966d6607544232e73341be0a7df8bc57efa51891457e864fb9d5cc954" in 2.427s (2.427s including waiting). Image size: 218068124 bytes.' metadata: creationTimestamp: "2026-07-02T13:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:23Z" name: model-chainer-raw-8e592-59dcb55855-b68v9.18be7da35be12ce1 namespace: kserve-ci-e2e-test resourceVersion: "15795" uid: c8822bd8-e0bb-4615-adca-f0a13cbf8455 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-174.ec2.internal source: component: kubelet host: ip-10-0-140-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-8e592} kind: Pod name: model-chainer-raw-8e592-59dcb55855-b68v9 namespace: kserve-ci-e2e-test resourceVersion: "15759" uid: 65330a8b-788f-4862-86f8-008592863d58 kind: Event lastTimestamp: "2026-07-02T13:57:23Z" message: 'Created container: model-chainer-raw-8e592' metadata: creationTimestamp: "2026-07-02T13:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:23Z" name: model-chainer-raw-8e592-59dcb55855-b68v9.18be7da35ea3eaf1 namespace: kserve-ci-e2e-test resourceVersion: "15796" uid: a802ec7a-69d3-4503-8d40-19442fe9fde0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-174.ec2.internal source: component: kubelet host: ip-10-0-140-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-8e592} kind: Pod name: model-chainer-raw-8e592-59dcb55855-b68v9 namespace: kserve-ci-e2e-test resourceVersion: "15759" uid: 65330a8b-788f-4862-86f8-008592863d58 kind: Event lastTimestamp: "2026-07-02T13:57:23Z" message: Started container model-chainer-raw-8e592 metadata: creationTimestamp: "2026-07-02T13:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:23Z" name: model-chainer-raw-8e592-59dcb55855-b68v9.18be7da35f0cc231 namespace: kserve-ci-e2e-test resourceVersion: "15797" uid: 7db26fc5-d471-4d1a-93e8-4768abb68ebb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-174.ec2.internal source: component: kubelet host: ip-10-0-140-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-8e592} kind: Pod name: model-chainer-raw-8e592-59dcb55855-b68v9 namespace: kserve-ci-e2e-test resourceVersion: "15759" uid: 65330a8b-788f-4862-86f8-008592863d58 kind: Event lastTimestamp: "2026-07-02T13:57:30Z" message: Stopping container model-chainer-raw-8e592 metadata: creationTimestamp: "2026-07-02T13:57:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:30Z" name: model-chainer-raw-8e592-59dcb55855-b68v9.18be7da50b63d30b namespace: kserve-ci-e2e-test resourceVersion: "15897" uid: 40f696cf-24ac-4e52-8fe9-60aa67071a05 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-174.ec2.internal source: component: kubelet host: ip-10-0-140-174.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:57:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-8e592} kind: Pod name: model-chainer-raw-8e592-59dcb55855-b68v9 namespace: kserve-ci-e2e-test resourceVersion: "15759" uid: 65330a8b-788f-4862-86f8-008592863d58 kind: Event lastTimestamp: "2026-07-02T13:57:59Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T13:57:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:59Z" name: model-chainer-raw-8e592-59dcb55855-b68v9.18be7da60f7a37bd namespace: kserve-ci-e2e-test resourceVersion: "16289" uid: 14927efc-ca9d-46aa-a2b6-06a89e8cd8c4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-174.ec2.internal source: component: kubelet host: ip-10-0-140-174.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-8e592-59dcb55855 namespace: kserve-ci-e2e-test resourceVersion: "15752" uid: 326200da-3b8b-4df9-b8b7-0800373e514b kind: Event lastTimestamp: "2026-07-02T13:57:20Z" message: 'Created pod: model-chainer-raw-8e592-59dcb55855-b68v9' metadata: creationTimestamp: "2026-07-02T13:57:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:57:20Z" name: model-chainer-raw-8e592-59dcb55855.18be7da2b02ef55d namespace: kserve-ci-e2e-test resourceVersion: "15757" uid: 39fe5998-2fc6-4ef7-8771-21e530188d5f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-8e592 namespace: kserve-ci-e2e-test resourceVersion: "15750" uid: 10adb0bb-0385-439d-8864-85de26e0d886 kind: Event lastTimestamp: "2026-07-02T13:57:20Z" message: Scaled up replica set model-chainer-raw-8e592-59dcb55855 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:57:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:57:20Z" name: model-chainer-raw-8e592.18be7da2aeef6222 namespace: kserve-ci-e2e-test resourceVersion: "15753" uid: 11d5192a-4414-445d-83ea-a8b19c238f4f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-8e592 namespace: kserve-ci-e2e-test resourceVersion: "15766" uid: 63de8a90-51e6-477c-8635-bbcc1a706305 kind: Event lastTimestamp: "2026-07-02T13:57:20Z" message: InferenceGraph [model-chainer-raw-8e592] is Ready metadata: creationTimestamp: "2026-07-02T13:57:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:57:20Z" name: model-chainer-raw-8e592.18be7da2b5f52892 namespace: kserve-ci-e2e-test resourceVersion: "15768" uid: fb0bb6be-d5c3-486d-8282-f520eb79c3a5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:59:10.637383Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-e16db-576b454748-phwkk namespace: kserve-ci-e2e-test resourceVersion: "16828" uid: dfe32740-497c-4ce8-bfcf-586d99636141 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-e16db-576b454748-phwkk to ip-10-0-140-174.ec2.internal metadata: creationTimestamp: "2026-07-02T13:59:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:59:10Z" name: model-chainer-raw-hpa-e16db-576b454748-phwkk.18be7dbc67ff11cc namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: eee52c91-dead-462c-a8fa-843f25aa6257 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8899f5b4d-w9vws source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:11Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-e16db-576b454748-phwkk namespace: kserve-ci-e2e-test resourceVersion: "16837" uid: dfe32740-497c-4ce8-bfcf-586d99636141 kind: Event lastTimestamp: "2026-07-02T13:59:11Z" message: Add eth0 [10.132.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:59:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:59:11Z" name: model-chainer-raw-hpa-e16db-576b454748-phwkk.18be7dbc80dc6007 namespace: kserve-ci-e2e-test resourceVersion: "16853" uid: c9e643b9-ba3d-4b4c-8d31-7de7880caec9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-e16db} kind: Pod name: model-chainer-raw-hpa-e16db-576b454748-phwkk namespace: kserve-ci-e2e-test resourceVersion: "16831" uid: dfe32740-497c-4ce8-bfcf-586d99636141 kind: Event lastTimestamp: "2026-07-02T13:59:11Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b87693b966d6607544232e73341be0a7df8bc57efa51891457e864fb9d5cc954" already present on machine metadata: creationTimestamp: "2026-07-02T13:59:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:11Z" name: model-chainer-raw-hpa-e16db-576b454748-phwkk.18be7dbc8242b48b namespace: kserve-ci-e2e-test resourceVersion: "16855" uid: b6d83326-89e2-4832-8f3f-ea51bf99a353 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-174.ec2.internal source: component: kubelet host: ip-10-0-140-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-e16db} kind: Pod name: model-chainer-raw-hpa-e16db-576b454748-phwkk namespace: kserve-ci-e2e-test resourceVersion: "16831" uid: dfe32740-497c-4ce8-bfcf-586d99636141 kind: Event lastTimestamp: "2026-07-02T13:59:11Z" message: 'Created container: model-chainer-raw-hpa-e16db' metadata: creationTimestamp: "2026-07-02T13:59:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:11Z" name: model-chainer-raw-hpa-e16db-576b454748-phwkk.18be7dbc84d10bd7 namespace: kserve-ci-e2e-test resourceVersion: "16856" uid: 00793f51-878b-431c-8274-d9ec252c67b9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-174.ec2.internal source: component: kubelet host: ip-10-0-140-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-e16db} kind: Pod name: model-chainer-raw-hpa-e16db-576b454748-phwkk namespace: kserve-ci-e2e-test resourceVersion: "16831" uid: dfe32740-497c-4ce8-bfcf-586d99636141 kind: Event lastTimestamp: "2026-07-02T13:59:11Z" message: Started container model-chainer-raw-hpa-e16db metadata: creationTimestamp: "2026-07-02T13:59:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:11Z" name: model-chainer-raw-hpa-e16db-576b454748-phwkk.18be7dbc853c0fe2 namespace: kserve-ci-e2e-test resourceVersion: "16857" uid: 74f3e189-9883-46f3-a3b4-b448401ca6a9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-174.ec2.internal source: component: kubelet host: ip-10-0-140-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-e16db} kind: Pod name: model-chainer-raw-hpa-e16db-576b454748-phwkk namespace: kserve-ci-e2e-test resourceVersion: "16831" uid: dfe32740-497c-4ce8-bfcf-586d99636141 kind: Event lastTimestamp: "2026-07-02T13:59:20Z" message: Stopping container model-chainer-raw-hpa-e16db metadata: creationTimestamp: "2026-07-02T13:59:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:20Z" name: model-chainer-raw-hpa-e16db-576b454748-phwkk.18be7dbebea2042e namespace: kserve-ci-e2e-test resourceVersion: "16942" uid: 704fd295-5466-434c-b001-432ede69f3d6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-174.ec2.internal source: component: kubelet host: ip-10-0-140-174.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:59:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-e16db} kind: Pod name: model-chainer-raw-hpa-e16db-576b454748-phwkk namespace: kserve-ci-e2e-test resourceVersion: "16831" uid: dfe32740-497c-4ce8-bfcf-586d99636141 kind: Event lastTimestamp: "2026-07-02T13:59:48Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T13:59:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:59:48Z" name: model-chainer-raw-hpa-e16db-576b454748-phwkk.18be7dbf499fe813 namespace: kserve-ci-e2e-test resourceVersion: "17324" uid: 706e26b1-916c-49fe-b5c9-5961d34adc12 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-174.ec2.internal source: component: kubelet host: ip-10-0-140-174.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-e16db-576b454748 namespace: kserve-ci-e2e-test resourceVersion: "16824" uid: b11e9d5c-5ba6-4998-813e-fe8e38ea8bea kind: Event lastTimestamp: "2026-07-02T13:59:10Z" message: 'Created pod: model-chainer-raw-hpa-e16db-576b454748-phwkk' metadata: creationTimestamp: "2026-07-02T13:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:59:10Z" name: model-chainer-raw-hpa-e16db-576b454748.18be7dbc67524577 namespace: kserve-ci-e2e-test resourceVersion: "16829" uid: 91003714-f5a4-45a9-873e-703072f23c96 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-e16db namespace: kserve-ci-e2e-test resourceVersion: "16822" uid: 9b5bc7db-7e7d-4268-9a1d-2a464e7b390e kind: Event lastTimestamp: "2026-07-02T13:59:10Z" message: Scaled up replica set model-chainer-raw-hpa-e16db-576b454748 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:59:10Z" name: model-chainer-raw-hpa-e16db.18be7dbc65637629 namespace: kserve-ci-e2e-test resourceVersion: "16825" uid: 084d6e38-a56e-4d47-afa2-e047c539484e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:59:10Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-e16db namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 9225b713-2f73-4aef-b806-eb31161fc445 kind: Event lastTimestamp: "2026-07-02T13:59:10Z" message: InferenceGraph [model-chainer-raw-hpa-e16db] is Ready metadata: creationTimestamp: "2026-07-02T13:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:59:10Z" name: model-chainer-raw-hpa-e16db.18be7dbc6c83ac86 namespace: kserve-ci-e2e-test resourceVersion: "16841" uid: 08eef18c-4d0c-42e0-89f1-c3b0a3420920 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:06:20.267654Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-e66b7-predictor-769988c997-zrpvc namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: 156661a7-3632-4237-ab09-f28babdc1b04 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-e66b7-predictor-769988c997-zrpvc to ip-10-0-136-2.ec2.internal metadata: creationTimestamp: "2026-07-02T14:06:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:06:20Z" name: raw-sklearn-e66b7-predictor-769988c997-zrpvc.18be7e206ff4cfdb namespace: kserve-ci-e2e-test resourceVersion: "20531" uid: 2a64405c-83eb-4a83-9a76-df9155132e7c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8899f5b4d-w9vws source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:20Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-e66b7-predictor-769988c997-zrpvc namespace: kserve-ci-e2e-test resourceVersion: "20535" uid: 156661a7-3632-4237-ab09-f28babdc1b04 kind: Event lastTimestamp: "2026-07-02T14:06:20Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:06:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:06:20Z" name: raw-sklearn-e66b7-predictor-769988c997-zrpvc.18be7e2089a096a1 namespace: kserve-ci-e2e-test resourceVersion: "20548" uid: de44524d-896b-4733-ae2a-1383dc400dac reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-e66b7-predictor-769988c997-zrpvc namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: 156661a7-3632-4237-ab09-f28babdc1b04 kind: Event lastTimestamp: "2026-07-02T14:06:20Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd2b86b0a2c9500aabe87a633c3b9bab30a880b19ff42859e99cc17f113ecdb0" already present on machine metadata: creationTimestamp: "2026-07-02T14:06:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:20Z" name: raw-sklearn-e66b7-predictor-769988c997-zrpvc.18be7e208b1bc482 namespace: kserve-ci-e2e-test resourceVersion: "20550" uid: ed5d5650-1e9c-494b-8118-7ab40897f670 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-e66b7-predictor-769988c997-zrpvc namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: 156661a7-3632-4237-ab09-f28babdc1b04 kind: Event lastTimestamp: "2026-07-02T14:06:20Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:06:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:20Z" name: raw-sklearn-e66b7-predictor-769988c997-zrpvc.18be7e208de95e75 namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 2fbb006b-d7ee-421f-afa1-e31bbf14f35e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-e66b7-predictor-769988c997-zrpvc namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: 156661a7-3632-4237-ab09-f28babdc1b04 kind: Event lastTimestamp: "2026-07-02T14:06:20Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:06:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:20Z" name: raw-sklearn-e66b7-predictor-769988c997-zrpvc.18be7e208e5ae180 namespace: kserve-ci-e2e-test resourceVersion: "20552" uid: 8984d807-24d8-4db3-ab0c-a803a8cf79fe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-e66b7-predictor-769988c997-zrpvc namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: 156661a7-3632-4237-ab09-f28babdc1b04 kind: Event lastTimestamp: "2026-07-02T14:06:25Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1695" already present on machine metadata: creationTimestamp: "2026-07-02T14:06:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:25Z" name: raw-sklearn-e66b7-predictor-769988c997-zrpvc.18be7e21ab8e3dfb namespace: kserve-ci-e2e-test resourceVersion: "20591" uid: 1b63f740-ff62-4fec-8eee-f745e10b1453 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-e66b7-predictor-769988c997-zrpvc namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: 156661a7-3632-4237-ab09-f28babdc1b04 kind: Event lastTimestamp: "2026-07-02T14:06:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:06:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:25Z" name: raw-sklearn-e66b7-predictor-769988c997-zrpvc.18be7e21aff7a4b2 namespace: kserve-ci-e2e-test resourceVersion: "20596" uid: abefdf83-222d-4883-bd02-08633e407e46 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-e66b7-predictor-769988c997-zrpvc namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: 156661a7-3632-4237-ab09-f28babdc1b04 kind: Event lastTimestamp: "2026-07-02T14:06:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:06:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:25Z" name: raw-sklearn-e66b7-predictor-769988c997-zrpvc.18be7e21b06d72a1 namespace: kserve-ci-e2e-test resourceVersion: "20597" uid: 27492da1-72c0-4754-b781-f7933c4dfe55 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-e66b7-predictor-769988c997-zrpvc namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: 156661a7-3632-4237-ab09-f28babdc1b04 kind: Event lastTimestamp: "2026-07-02T14:06:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:06:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:25Z" name: raw-sklearn-e66b7-predictor-769988c997-zrpvc.18be7e21b0778636 namespace: kserve-ci-e2e-test resourceVersion: "20598" uid: e4129b1b-8601-4935-80fd-3ece6910131f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-e66b7-predictor-769988c997-zrpvc namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: 156661a7-3632-4237-ab09-f28babdc1b04 kind: Event lastTimestamp: "2026-07-02T14:06:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:06:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:25Z" name: raw-sklearn-e66b7-predictor-769988c997-zrpvc.18be7e21b3b2bd8d namespace: kserve-ci-e2e-test resourceVersion: "20600" uid: c98b8443-a6bc-43d9-a67c-48ba106c5f84 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-e66b7-predictor-769988c997-zrpvc namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: 156661a7-3632-4237-ab09-f28babdc1b04 kind: Event lastTimestamp: "2026-07-02T14:06:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:06:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:25Z" name: raw-sklearn-e66b7-predictor-769988c997-zrpvc.18be7e21b421706f namespace: kserve-ci-e2e-test resourceVersion: "20601" uid: 7265a47b-48c8-4410-adb7-57a5c7bee7d6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T14:06:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-e66b7-predictor-769988c997-zrpvc namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: 156661a7-3632-4237-ab09-f28babdc1b04 kind: Event lastTimestamp: "2026-07-02T14:07:53Z" message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:07:53Z" name: raw-sklearn-e66b7-predictor-769988c997-zrpvc.18be7e2223780e41 namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: e1e1ec47-d6d6-485b-83f9-027394c9f6a5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-e66b7-predictor-769988c997-zrpvc namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: 156661a7-3632-4237-ab09-f28babdc1b04 kind: Event lastTimestamp: "2026-07-02T14:07:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:07:50Z" name: raw-sklearn-e66b7-predictor-769988c997-zrpvc.18be7e356e1dc9ea namespace: kserve-ci-e2e-test resourceVersion: "21218" uid: 712abda0-2d3b-4e46-8362-cd05aed518b5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-e66b7-predictor-769988c997-zrpvc namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: 156661a7-3632-4237-ab09-f28babdc1b04 kind: Event lastTimestamp: "2026-07-02T14:07:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:07:50Z" name: raw-sklearn-e66b7-predictor-769988c997-zrpvc.18be7e356e1e6a7c namespace: kserve-ci-e2e-test resourceVersion: "21219" uid: aee726cd-f1b0-4e19-ad8a-d722999d9c6c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-e66b7-predictor-769988c997-zrpvc namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: 156661a7-3632-4237-ab09-f28babdc1b04 kind: Event lastTimestamp: "2026-07-02T14:07:53Z" message: 'Readiness probe failed: Get "https://10.133.0.38:8643/healthz": dial tcp 10.133.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:07:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:07:53Z" name: raw-sklearn-e66b7-predictor-769988c997-zrpvc.18be7e3629aa7949 namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 1351cb2a-24c3-407d-9a03-eeafb7240b87 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-e66b7-predictor-769988c997 namespace: kserve-ci-e2e-test resourceVersion: "20516" uid: e0684cf3-3625-4793-bff3-ca7847c7e412 kind: Event lastTimestamp: "2026-07-02T14:06:20Z" message: 'Created pod: raw-sklearn-e66b7-predictor-769988c997-zrpvc' metadata: creationTimestamp: "2026-07-02T14:06:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:06:20Z" name: raw-sklearn-e66b7-predictor-769988c997.18be7e206f7cc4f0 namespace: kserve-ci-e2e-test resourceVersion: "20528" uid: c24882f0-ba08-46d4-ba7f-23d33059092a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-e66b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "20514" uid: 1dbd52d6-fbf4-409c-be4d-a5c399ef1736 kind: Event lastTimestamp: "2026-07-02T14:06:20Z" message: Scaled up replica set raw-sklearn-e66b7-predictor-769988c997 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:06:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:06:20Z" name: raw-sklearn-e66b7-predictor.18be7e206c483300 namespace: kserve-ci-e2e-test resourceVersion: "20517" uid: af1a4801-c11d-409c-86a8-fb90881471ca reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:06:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-e66b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "20515" uid: a57c7102-ebae-496b-81ea-277d72e816d2 kind: Event lastTimestamp: "2026-07-02T14:07:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:06:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:07:05Z" name: raw-sklearn-e66b7-predictor.18be7e23eaf9e21e namespace: kserve-ci-e2e-test resourceVersion: "20883" uid: a83508e2-0f18-4707-bb7f-b0577fc2c3a8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:06:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-e66b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "20515" uid: a57c7102-ebae-496b-81ea-277d72e816d2 kind: Event lastTimestamp: "2026-07-02T14:07:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:06:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:07:05Z" name: raw-sklearn-e66b7-predictor.18be7e23eb63d964 namespace: kserve-ci-e2e-test resourceVersion: "20884" uid: 27b331c6-ce5d-4488-99cc-717cac66f09c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:07:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-e66b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "20672" uid: a57c7102-ebae-496b-81ea-277d72e816d2 kind: Event lastTimestamp: "2026-07-02T14:07:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:07:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:07:50Z" name: raw-sklearn-e66b7-predictor.18be7e2e69f81cab namespace: kserve-ci-e2e-test resourceVersion: "21201" uid: 6473e251-41db-474f-8877-a1274da5c234 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:07:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-e66b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "20672" uid: a57c7102-ebae-496b-81ea-277d72e816d2 kind: Event lastTimestamp: "2026-07-02T14:07:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:07:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:07:50Z" name: raw-sklearn-e66b7-predictor.18be7e2e6a6b8b3e namespace: kserve-ci-e2e-test resourceVersion: "21202" uid: 3f39e917-6784-4e52-837c-dd9771f2d0c7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T14:06:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-e66b7 namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: 239367fc-179e-4de3-9586-7fc465c99609 kind: Event lastTimestamp: "2026-07-02T14:07:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:06:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:07:43Z" name: raw-sklearn-e66b7.18be7e206c4dcd9c namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: 56b0e56f-cb35-40c8-aa7c-96a1b664cbf0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-e66b7 namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: 239367fc-179e-4de3-9586-7fc465c99609 kind: Event lastTimestamp: "2026-07-02T14:06:20Z" message: 'Failed to update status for InferenceService "raw-sklearn-e66b7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-e66b7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:06:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:06:20Z" name: raw-sklearn-e66b7.18be7e206cc9c80b namespace: kserve-ci-e2e-test resourceVersion: "20520" uid: 320c344f-8a69-4c93-a585-0d6b7034445c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-e66b7 namespace: kserve-ci-e2e-test resourceVersion: "21156" uid: 239367fc-179e-4de3-9586-7fc465c99609 kind: Event lastTimestamp: "2026-07-02T14:07:43Z" message: InferenceService [raw-sklearn-e66b7] is Ready metadata: creationTimestamp: "2026-07-02T14:07:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:07:43Z" name: raw-sklearn-e66b7.18be7e33db53dfde namespace: kserve-ci-e2e-test resourceVersion: "21157" uid: 8a5cdc63-91f0-4a13-9057-0621a3314076 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:07:55.579119Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x namespace: kserve-ci-e2e-test resourceVersion: "21287" uid: 31db5049-4175-4b09-a076-632b10322ecc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x to ip-10-0-136-2.ec2.internal metadata: creationTimestamp: "2026-07-02T14:07:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:07:55Z" name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x.18be7e36a0f66072 namespace: kserve-ci-e2e-test resourceVersion: "21291" uid: 34889d5c-a490-47f0-8968-475e5aa242e5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8899f5b4d-w9vws source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:56Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x namespace: kserve-ci-e2e-test resourceVersion: "21295" uid: 31db5049-4175-4b09-a076-632b10322ecc kind: Event lastTimestamp: "2026-07-02T14:07:56Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:07:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:07:56Z" name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x.18be7e36ba86d73b namespace: kserve-ci-e2e-test resourceVersion: "21307" uid: ef2d4a46-61c2-47f1-a09d-6be48184f4a2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 31db5049-4175-4b09-a076-632b10322ecc kind: Event lastTimestamp: "2026-07-02T14:07:56Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd2b86b0a2c9500aabe87a633c3b9bab30a880b19ff42859e99cc17f113ecdb0" already present on machine metadata: creationTimestamp: "2026-07-02T14:07:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:07:56Z" name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x.18be7e36bbf3d6e3 namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: 03e27de8-d939-4ed4-9414-876b22dbfcc5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 31db5049-4175-4b09-a076-632b10322ecc kind: Event lastTimestamp: "2026-07-02T14:07:56Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:07:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:07:56Z" name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x.18be7e36bebf30bd namespace: kserve-ci-e2e-test resourceVersion: "21312" uid: ea887926-c73b-45dd-a7d3-23a5166acf03 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 31db5049-4175-4b09-a076-632b10322ecc kind: Event lastTimestamp: "2026-07-02T14:07:56Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:07:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:07:56Z" name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x.18be7e36bf2d58e6 namespace: kserve-ci-e2e-test resourceVersion: "21313" uid: ec2b4a1a-21ca-43dd-9a70-e533c79930e8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 31db5049-4175-4b09-a076-632b10322ecc kind: Event lastTimestamp: "2026-07-02T14:07:59Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1695" already present on machine metadata: creationTimestamp: "2026-07-02T14:07:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:07:59Z" name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x.18be7e37a7a92644 namespace: kserve-ci-e2e-test resourceVersion: "21341" uid: 4df64b48-c0d4-42c0-b14c-d35db690750d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:08:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 31db5049-4175-4b09-a076-632b10322ecc kind: Event lastTimestamp: "2026-07-02T14:08:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:08:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:08:00Z" name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x.18be7e37aaf60f3a namespace: kserve-ci-e2e-test resourceVersion: "21344" uid: 92af7785-dad2-4de1-ad3c-826f359976de reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:08:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 31db5049-4175-4b09-a076-632b10322ecc kind: Event lastTimestamp: "2026-07-02T14:08:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:08:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:08:00Z" name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x.18be7e37ab646d34 namespace: kserve-ci-e2e-test resourceVersion: "21345" uid: a9982f37-0a5f-4acc-b2ee-68dae2811ad1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:08:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 31db5049-4175-4b09-a076-632b10322ecc kind: Event lastTimestamp: "2026-07-02T14:08:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:08:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:08:00Z" name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x.18be7e37ab6d60ca namespace: kserve-ci-e2e-test resourceVersion: "21347" uid: b2ba6c22-1dda-47b7-ae6d-ca538817871b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:08:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 31db5049-4175-4b09-a076-632b10322ecc kind: Event lastTimestamp: "2026-07-02T14:08:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:08:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:08:00Z" name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x.18be7e37ae3c79d3 namespace: kserve-ci-e2e-test resourceVersion: "21350" uid: 126e8053-7bff-47f7-bbdb-15cc8cd1967f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:08:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 31db5049-4175-4b09-a076-632b10322ecc kind: Event lastTimestamp: "2026-07-02T14:08:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:08:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:08:00Z" name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x.18be7e37aea2bef7 namespace: kserve-ci-e2e-test resourceVersion: "21351" uid: 6d21b784-fd7e-495b-83d4-08237ea2876f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T14:08:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 31db5049-4175-4b09-a076-632b10322ecc kind: Event lastTimestamp: "2026-07-02T14:09:07Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:08:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:09:07Z" name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x.18be7e37e3af67cc namespace: kserve-ci-e2e-test resourceVersion: "21830" uid: ba09960e-512b-4974-99e9-32edc38e763c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:09:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 31db5049-4175-4b09-a076-632b10322ecc kind: Event lastTimestamp: "2026-07-02T14:09:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:09:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:09:25Z" name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x.18be7e4b9da7405a namespace: kserve-ci-e2e-test resourceVersion: "21990" uid: 58019d43-64a9-444d-ac84-7fbac52c6e8e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:09:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 31db5049-4175-4b09-a076-632b10322ecc kind: Event lastTimestamp: "2026-07-02T14:09:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:09:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:09:25Z" name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x.18be7e4b9da7cf92 namespace: kserve-ci-e2e-test resourceVersion: "21991" uid: 1e742454-7988-4d1d-be0d-f39b0500874c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:09:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: 31db5049-4175-4b09-a076-632b10322ecc kind: Event lastTimestamp: "2026-07-02T14:09:26Z" message: 'Readiness probe failed: Get "https://10.133.0.39:8643/healthz": dial tcp 10.133.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:09:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:09:26Z" name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x.18be7e4be9ecbe89 namespace: kserve-ci-e2e-test resourceVersion: "22001" uid: 5c91328c-9a16-4634-9112-b26db0283fc5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-2.ec2.internal source: component: kubelet host: ip-10-0-136-2.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7 namespace: kserve-ci-e2e-test resourceVersion: "21276" uid: 1d178dab-4a8e-4794-b4b9-56ee55f9454a kind: Event lastTimestamp: "2026-07-02T14:07:55Z" message: 'Created pod: raw-sklearn-runtime-9e884-predictor-bf9c64fc7-78b6x' metadata: creationTimestamp: "2026-07-02T14:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:07:55Z" name: raw-sklearn-runtime-9e884-predictor-bf9c64fc7.18be7e36a0872fe0 namespace: kserve-ci-e2e-test resourceVersion: "21288" uid: 005587bf-2dff-46e7-b017-61ffe9b38637 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-9e884-predictor namespace: kserve-ci-e2e-test resourceVersion: "21274" uid: c53553bd-3f8f-4165-83bf-a6abf6f01db7 kind: Event lastTimestamp: "2026-07-02T14:07:55Z" message: Scaled up replica set raw-sklearn-runtime-9e884-predictor-bf9c64fc7 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:07:55Z" name: raw-sklearn-runtime-9e884-predictor.18be7e369bda44be namespace: kserve-ci-e2e-test resourceVersion: "21277" uid: 26e47529-033b-4bb7-a264-d4507a63efc5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:08:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-9e884-predictor namespace: kserve-ci-e2e-test resourceVersion: "21275" uid: 5eb04a57-87da-49b8-a2ed-a9241d29e603 kind: Event lastTimestamp: "2026-07-02T14:08:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:08:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:08:40Z" name: raw-sklearn-runtime-9e884-predictor.18be7e3a1b16cb33 namespace: kserve-ci-e2e-test resourceVersion: "21641" uid: a07c752e-4aed-4021-8514-77e45b8cb03a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:08:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-9e884-predictor namespace: kserve-ci-e2e-test resourceVersion: "21275" uid: 5eb04a57-87da-49b8-a2ed-a9241d29e603 kind: Event lastTimestamp: "2026-07-02T14:08:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:08:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:08:40Z" name: raw-sklearn-runtime-9e884-predictor.18be7e3a1b93076c namespace: kserve-ci-e2e-test resourceVersion: "21643" uid: d9b0b895-63ad-4d6c-a147-3d87d81b955a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:08:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-9e884-predictor namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 5eb04a57-87da-49b8-a2ed-a9241d29e603 kind: Event lastTimestamp: "2026-07-02T14:09:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:09:25Z" name: raw-sklearn-runtime-9e884-predictor.18be7e3d9a60d9ed namespace: kserve-ci-e2e-test resourceVersion: "21973" uid: 21bc1ae4-c20d-4e0b-b9c9-e5d99f589f96 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:08:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-9e884-predictor namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 5eb04a57-87da-49b8-a2ed-a9241d29e603 kind: Event lastTimestamp: "2026-07-02T14:09:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:08:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:09:25Z" name: raw-sklearn-runtime-9e884-predictor.18be7e3d9acb855b namespace: kserve-ci-e2e-test resourceVersion: "21975" uid: 86730414-b264-4d12-afc0-6c66b05597ee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T14:07:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-9e884 namespace: kserve-ci-e2e-test resourceVersion: "21267" uid: 4535508e-046d-4393-8fb0-c727b9178289 kind: Event lastTimestamp: "2026-07-02T14:09:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:09:15Z" name: raw-sklearn-runtime-9e884.18be7e369bde1389 namespace: kserve-ci-e2e-test resourceVersion: "21900" uid: a61beab0-0b65-4a33-abba-5c5ff632b70b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:07:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-9e884 namespace: kserve-ci-e2e-test resourceVersion: "21267" uid: 4535508e-046d-4393-8fb0-c727b9178289 kind: Event lastTimestamp: "2026-07-02T14:07:55Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-9e884": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-9e884": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:07:55Z" name: raw-sklearn-runtime-9e884.18be7e369c79c182 namespace: kserve-ci-e2e-test resourceVersion: "21297" uid: 15ea5757-f4c2-4614-a038-e43c67a59fae reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-9e884 namespace: kserve-ci-e2e-test resourceVersion: "21269" uid: 4535508e-046d-4393-8fb0-c727b9178289 kind: Event lastTimestamp: "2026-07-02T14:07:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-9e884": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:07:55Z" name: raw-sklearn-runtime-9e884.18be7e36a32a769c namespace: kserve-ci-e2e-test resourceVersion: "21298" uid: 2aedb5ae-e971-4e1b-b24a-10e082519a90 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:09:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-9e884 namespace: kserve-ci-e2e-test resourceVersion: "21896" uid: 4535508e-046d-4393-8fb0-c727b9178289 kind: Event lastTimestamp: "2026-07-02T14:09:15Z" message: InferenceService [raw-sklearn-runtime-9e884] is Ready metadata: creationTimestamp: "2026-07-02T14:09:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:09:15Z" name: raw-sklearn-runtime-9e884.18be7e4945752d16 namespace: kserve-ci-e2e-test resourceVersion: "21898" uid: 630c9a65-e0ec-44ce-95fd-66d9adac47c8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "22246"