--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:19:01.749086Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-89de80-predictor-555565cf89-qcg8j namespace: kserve-ci-e2e-test resourceVersion: "20450" uid: ea47f1f4-0c6e-475f-b8c9-84404de6dbd5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-89de80-predictor-555565cf89-qcg8j to ip-10-0-143-168.ec2.internal metadata: creationTimestamp: "2026-07-07T15:19:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:19:01Z" name: isvc-init-fail-89de80-predictor-555565cf89-qcg8j.18c00afec30cf8ca namespace: kserve-ci-e2e-test resourceVersion: "20456" uid: 94cb0385-3e7e-4a9a-bd89-c9f7cd63f920 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-77985f99fc-vc8tw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-89de80-predictor-555565cf89-qcg8j namespace: kserve-ci-e2e-test resourceVersion: "20459" uid: ea47f1f4-0c6e-475f-b8c9-84404de6dbd5 kind: Event lastTimestamp: "2026-07-07T15:19:02Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:19:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:19:02Z" name: isvc-init-fail-89de80-predictor-555565cf89-qcg8j.18c00afedd2cb8df namespace: kserve-ci-e2e-test resourceVersion: "20471" uid: db1c2b63-926b-4be0-bf08-ffc1d8519656 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:19:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-89de80-predictor-555565cf89-qcg8j namespace: kserve-ci-e2e-test resourceVersion: "20451" uid: ea47f1f4-0c6e-475f-b8c9-84404de6dbd5 kind: Event lastTimestamp: "2026-07-07T15:19:06Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:09d6f486359107a6e5d5aadd7d4fa2ea151504e2bf98f31f8912a941eb631c4b" already present on machine metadata: creationTimestamp: "2026-07-07T15:19:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:19:06Z" name: isvc-init-fail-89de80-predictor-555565cf89-qcg8j.18c00afedf0fc186 namespace: kserve-ci-e2e-test resourceVersion: "20511" uid: cac6e2a0-f1f6-4358-8290-3bdfb4afe448 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:19:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-89de80-predictor-555565cf89-qcg8j namespace: kserve-ci-e2e-test resourceVersion: "20451" uid: ea47f1f4-0c6e-475f-b8c9-84404de6dbd5 kind: Event lastTimestamp: "2026-07-07T15:19:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T15:19:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:19:06Z" name: isvc-init-fail-89de80-predictor-555565cf89-qcg8j.18c00afee1dcc958 namespace: kserve-ci-e2e-test resourceVersion: "20516" uid: 8d4d9f28-216d-4e23-81ed-1c08d039becb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:19:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-89de80-predictor-555565cf89-qcg8j namespace: kserve-ci-e2e-test resourceVersion: "20451" uid: ea47f1f4-0c6e-475f-b8c9-84404de6dbd5 kind: Event lastTimestamp: "2026-07-07T15:19:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T15:19:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:19:06Z" name: isvc-init-fail-89de80-predictor-555565cf89-qcg8j.18c00afee2521174 namespace: kserve-ci-e2e-test resourceVersion: "20517" uid: b908d45e-76ba-47d5-9983-d24652a16771 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-89de80-predictor-555565cf89-qcg8j namespace: kserve-ci-e2e-test resourceVersion: "20451" uid: ea47f1f4-0c6e-475f-b8c9-84404de6dbd5 kind: Event lastTimestamp: "2026-07-07T15:19:09Z" message: Back-off restarting failed container storage-initializer in pod isvc-init-fail-89de80-predictor-555565cf89-qcg8j_kserve-ci-e2e-test(ea47f1f4-0c6e-475f-b8c9-84404de6dbd5) metadata: creationTimestamp: "2026-07-07T15:19:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:19:09Z" name: isvc-init-fail-89de80-predictor-555565cf89-qcg8j.18c00b009dc5fafc namespace: kserve-ci-e2e-test resourceVersion: "20535" uid: d4cdff16-096b-416f-8f0a-7f94311303fb reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-89de80-predictor-555565cf89 namespace: kserve-ci-e2e-test resourceVersion: "20441" uid: beb33eb7-bc60-41e0-946b-a076f22b6e7d kind: Event lastTimestamp: "2026-07-07T15:19:01Z" message: 'Created pod: isvc-init-fail-89de80-predictor-555565cf89-qcg8j' metadata: creationTimestamp: "2026-07-07T15:19:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:19:01Z" name: isvc-init-fail-89de80-predictor-555565cf89.18c00afec2974f41 namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: ac748edb-691b-4c7f-8ddc-528e06dff01b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-89de80-predictor namespace: kserve-ci-e2e-test resourceVersion: "20439" uid: c604e6ee-1ab2-42e7-b514-20f47418b98c kind: Event lastTimestamp: "2026-07-07T15:19:01Z" message: Scaled up replica set isvc-init-fail-89de80-predictor-555565cf89 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:19:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:19:01Z" name: isvc-init-fail-89de80-predictor.18c00afebf3d6d74 namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: 5c01388f-8a40-4745-a90b-09251ab363a5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-07-07T15:19:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-89de80 namespace: kserve-ci-e2e-test resourceVersion: "20432" uid: f3eb4f0f-7446-4959-a82d-eddc7a6b3018 kind: Event lastTimestamp: "2026-07-07T15:19:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:19:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:19:09Z" name: isvc-init-fail-89de80.18c00afebf6c9f59 namespace: kserve-ci-e2e-test resourceVersion: "20540" uid: 5c6b433b-aea8-4c0d-af4b-d0afc44ccb18 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-89de80 namespace: kserve-ci-e2e-test resourceVersion: "20432" uid: f3eb4f0f-7446-4959-a82d-eddc7a6b3018 kind: Event lastTimestamp: "2026-07-07T15:19:01Z" message: 'Failed to update status for InferenceService "isvc-init-fail-89de80": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-89de80": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:19:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:19:01Z" name: isvc-init-fail-89de80.18c00afebffeb9c7 namespace: kserve-ci-e2e-test resourceVersion: "20447" uid: 48697ec9-67a6-4dc0-801a-c25bfa696f27 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-89de80 namespace: kserve-ci-e2e-test resourceVersion: "20432" uid: f3eb4f0f-7446-4959-a82d-eddc7a6b3018 kind: Event lastTimestamp: "2026-07-07T15:19:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-89de80": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:19:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:19:01Z" name: isvc-init-fail-89de80.18c00afebffef47f namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: 968bc2e4-adc3-4003-aad2-3cf2b86ceddd reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:12:37.666430Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9 namespace: kserve-ci-e2e-test resourceVersion: "17334" uid: 066b718a-05bd-447a-b270-d84635d62900 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9 to ip-10-0-143-168.ec2.internal metadata: creationTimestamp: "2026-07-07T15:12:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:12:37Z" name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9.18c00aa555f0bc46 namespace: kserve-ci-e2e-test resourceVersion: "17338" uid: 3539b2f3-0542-4444-a5f6-5509b35c25dc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-77985f99fc-vc8tw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:37Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9 namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: 066b718a-05bd-447a-b270-d84635d62900 kind: Event lastTimestamp: "2026-07-07T15:12:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-logger-raw-c27ac-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T15:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:37Z" name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9.18c00aa5600497d8 namespace: kserve-ci-e2e-test resourceVersion: "17348" uid: 0c000003-cc7d-4144-9bc7-b991fbeaae9d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:38Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9 namespace: kserve-ci-e2e-test resourceVersion: "17344" uid: 066b718a-05bd-447a-b270-d84635d62900 kind: Event lastTimestamp: "2026-07-07T15:12:38Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:12:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:12:38Z" name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9.18c00aa5934e4dc7 namespace: kserve-ci-e2e-test resourceVersion: "17357" uid: fe01c75c-8016-4f3d-bb8e-2e88d723e11f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9 namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: 066b718a-05bd-447a-b270-d84635d62900 kind: Event lastTimestamp: "2026-07-07T15:12:38Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:09d6f486359107a6e5d5aadd7d4fa2ea151504e2bf98f31f8912a941eb631c4b" already present on machine metadata: creationTimestamp: "2026-07-07T15:12:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:38Z" name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9.18c00aa59502464c namespace: kserve-ci-e2e-test resourceVersion: "17360" uid: d918f6be-b49c-43e5-902b-39858fc02886 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9 namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: 066b718a-05bd-447a-b270-d84635d62900 kind: Event lastTimestamp: "2026-07-07T15:12:38Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T15:12:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:38Z" name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9.18c00aa59798b35c namespace: kserve-ci-e2e-test resourceVersion: "17361" uid: 1d875df7-d2a1-4793-be61-697831342322 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9 namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: 066b718a-05bd-447a-b270-d84635d62900 kind: Event lastTimestamp: "2026-07-07T15:12:38Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T15:12:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:38Z" name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9.18c00aa5982b6c2b namespace: kserve-ci-e2e-test resourceVersion: "17362" uid: e9f4c50a-686c-4a77-8281-a7066bdfd05c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9 namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: 066b718a-05bd-447a-b270-d84635d62900 kind: Event lastTimestamp: "2026-07-07T15:12:42Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:12:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:42Z" name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9.18c00aa6924de11d namespace: kserve-ci-e2e-test resourceVersion: "17399" uid: 28d790de-bb27-4d59-b6f2-6c570cb79c89 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9 namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: 066b718a-05bd-447a-b270-d84635d62900 kind: Event lastTimestamp: "2026-07-07T15:12:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:43Z" name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9.18c00aa694eddea4 namespace: kserve-ci-e2e-test resourceVersion: "17402" uid: 5ed8bb1f-13ac-4d53-9609-29b432842682 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9 namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: 066b718a-05bd-447a-b270-d84635d62900 kind: Event lastTimestamp: "2026-07-07T15:12:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:43Z" name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9.18c00aa69553b940 namespace: kserve-ci-e2e-test resourceVersion: "17403" uid: efb8d19d-16a8-4b72-bcc1-4f0d94635ef6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9 namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: 066b718a-05bd-447a-b270-d84635d62900 kind: Event lastTimestamp: "2026-07-07T15:12:43Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:43Z" name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9.18c00aa6955e813b namespace: kserve-ci-e2e-test resourceVersion: "17404" uid: 1f22e861-e163-4bd4-a987-b400801e176a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9 namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: 066b718a-05bd-447a-b270-d84635d62900 kind: Event lastTimestamp: "2026-07-07T15:12:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:43Z" name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9.18c00aa6987c8eb8 namespace: kserve-ci-e2e-test resourceVersion: "17409" uid: 1b249bfe-4e17-4ba5-b9aa-cd61973ccf43 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9 namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: 066b718a-05bd-447a-b270-d84635d62900 kind: Event lastTimestamp: "2026-07-07T15:12:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:43Z" name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9.18c00aa698e47c20 namespace: kserve-ci-e2e-test resourceVersion: "17410" uid: f299fdb2-86c6-4521-9d99-bb03fa5b9ed3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9 namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: 066b718a-05bd-447a-b270-d84635d62900 kind: Event lastTimestamp: "2026-07-07T15:12:43Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:c2ec3c7b4f4e1e45c65c5dc6bb443d6d9f93bbd95edfb128e24e40a56ed9bf66" already present on machine metadata: creationTimestamp: "2026-07-07T15:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:43Z" name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9.18c00aa698ef45bb namespace: kserve-ci-e2e-test resourceVersion: "17411" uid: 19ffba73-ae72-4b8e-8a3a-56cd9fd7881d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9 namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: 066b718a-05bd-447a-b270-d84635d62900 kind: Event lastTimestamp: "2026-07-07T15:12:43Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-07T15:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:43Z" name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9.18c00aa69ba67f20 namespace: kserve-ci-e2e-test resourceVersion: "17413" uid: 4744dbc6-9d63-4d61-b930-0f5a1f66cfec reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9 namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: 066b718a-05bd-447a-b270-d84635d62900 kind: Event lastTimestamp: "2026-07-07T15:12:43Z" message: Started container agent metadata: creationTimestamp: "2026-07-07T15:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:43Z" name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9.18c00aa69c0fbc61 namespace: kserve-ci-e2e-test resourceVersion: "17414" uid: f5b0123a-f7a2-4e18-804e-a5b249f99c62 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T15:12:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9 namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: 066b718a-05bd-447a-b270-d84635d62900 kind: Event lastTimestamp: "2026-07-07T15:14:20Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:14:21Z" name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9.18c00aa70a4d8668 namespace: kserve-ci-e2e-test resourceVersion: "18230" uid: 7a24c197-d652-4124-86e5-3939cd6ec754 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T15:12:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9 namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: 066b718a-05bd-447a-b270-d84635d62900 kind: Event lastTimestamp: "2026-07-07T15:14:20Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T15:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:14:21Z" name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9.18c00aa70a5a7f7d namespace: kserve-ci-e2e-test resourceVersion: "18231" uid: 7531ba35-7d98-4e75-ad5b-27079a6b1126 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9 namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: 066b718a-05bd-447a-b270-d84635d62900 kind: Event lastTimestamp: "2026-07-07T15:14:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:14:13Z" name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9.18c00abb89f7689a namespace: kserve-ci-e2e-test resourceVersion: "18118" uid: 1ae9acb0-01fd-4189-bc5a-ba6230abd1ea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9 namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: 066b718a-05bd-447a-b270-d84635d62900 kind: Event lastTimestamp: "2026-07-07T15:14:13Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-07T15:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:14:13Z" name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9.18c00abb89f7a9c0 namespace: kserve-ci-e2e-test resourceVersion: "18127" uid: c1cd54c8-c52a-4893-a716-86860154146f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9 namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: 066b718a-05bd-447a-b270-d84635d62900 kind: Event lastTimestamp: "2026-07-07T15:14:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:14:13Z" name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9.18c00abb89f7c752 namespace: kserve-ci-e2e-test resourceVersion: "18123" uid: c20ba6f8-de3a-4669-b685-9fc54f5790ac reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T15:14:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9 namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: 066b718a-05bd-447a-b270-d84635d62900 kind: Event lastTimestamp: "2026-07-07T15:14:30Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:14:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:14:30Z" name: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9.18c00abc3a902db2 namespace: kserve-ci-e2e-test resourceVersion: "18306" uid: 56dd3c77-53b2-4052-8e60-2b472271b50a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-c27ac-predictor-54d7f5d474 namespace: kserve-ci-e2e-test resourceVersion: "17324" uid: 25ab7eac-009a-4a6b-a929-df9666c28626 kind: Event lastTimestamp: "2026-07-07T15:12:37Z" message: 'Created pod: isvc-logger-raw-c27ac-predictor-54d7f5d474-l7ml9' metadata: creationTimestamp: "2026-07-07T15:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:12:37Z" name: isvc-logger-raw-c27ac-predictor-54d7f5d474.18c00aa555826d20 namespace: kserve-ci-e2e-test resourceVersion: "17335" uid: e734b0b8-4a61-4800-8b37-675c6f3a7303 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-c27ac-predictor namespace: kserve-ci-e2e-test resourceVersion: "17322" uid: 705befd5-15ff-4ba5-b60d-3e37c2af3522 kind: Event lastTimestamp: "2026-07-07T15:12:37Z" message: Scaled up replica set isvc-logger-raw-c27ac-predictor-54d7f5d474 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:12:37Z" name: isvc-logger-raw-c27ac-predictor.18c00aa5524fd141 namespace: kserve-ci-e2e-test resourceVersion: "17325" uid: fcbe9e04-9ab2-47a1-8a62-44bada13b086 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:12:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-c27ac-predictor namespace: kserve-ci-e2e-test resourceVersion: "17323" uid: f2f0873a-2676-4876-8e02-ce1169c84bc5 kind: Event lastTimestamp: "2026-07-07T15:13:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:12:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:13:07Z" name: isvc-logger-raw-c27ac-predictor.18c00aa8d0e033f4 namespace: kserve-ci-e2e-test resourceVersion: "17599" uid: a7f4a2d2-09f3-46d9-b49e-6f5ac0b216fd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:12:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-c27ac-predictor namespace: kserve-ci-e2e-test resourceVersion: "17323" uid: f2f0873a-2676-4876-8e02-ce1169c84bc5 kind: Event lastTimestamp: "2026-07-07T15:13:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:12:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:13:07Z" name: isvc-logger-raw-c27ac-predictor.18c00aa8d15e261b namespace: kserve-ci-e2e-test resourceVersion: "17600" uid: c7b95169-4d61-4478-8cae-939805d071fb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T15:13:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-c27ac-predictor namespace: kserve-ci-e2e-test resourceVersion: "17484" uid: f2f0873a-2676-4876-8e02-ce1169c84bc5 kind: Event lastTimestamp: "2026-07-07T15:14:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:14:07Z" name: isvc-logger-raw-c27ac-predictor.18c00aafcf92f743 namespace: kserve-ci-e2e-test resourceVersion: "18035" uid: f3766508-17c5-450f-8de0-1054f184136f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T15:13:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-c27ac-predictor namespace: kserve-ci-e2e-test resourceVersion: "17484" uid: f2f0873a-2676-4876-8e02-ce1169c84bc5 kind: Event lastTimestamp: "2026-07-07T15:14:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:14:07Z" name: isvc-logger-raw-c27ac-predictor.18c00aafd0188b26 namespace: kserve-ci-e2e-test resourceVersion: "18036" uid: 8c4ab5c5-15a6-438f-9ec6-14928ea09e0d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T15:12:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-c27ac namespace: kserve-ci-e2e-test resourceVersion: "17313" uid: a89d1228-e33e-49c0-b92c-153552d7023c kind: Event lastTimestamp: "2026-07-07T15:14:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:14:01Z" name: isvc-logger-raw-c27ac.18c00aa5524f7a80 namespace: kserve-ci-e2e-test resourceVersion: "17989" uid: 6df8aee4-5f0a-4ce1-ada6-f4a57aff47c6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-c27ac namespace: kserve-ci-e2e-test resourceVersion: "17313" uid: a89d1228-e33e-49c0-b92c-153552d7023c kind: Event lastTimestamp: "2026-07-07T15:12:37Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-c27ac": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-c27ac": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:12:37Z" name: isvc-logger-raw-c27ac.18c00aa552a7b6c0 namespace: kserve-ci-e2e-test resourceVersion: "17328" uid: 919f9d6c-a914-4c5f-9d7a-a0592742e153 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-c27ac namespace: kserve-ci-e2e-test resourceVersion: "17986" uid: a89d1228-e33e-49c0-b92c-153552d7023c kind: Event lastTimestamp: "2026-07-07T15:14:01Z" message: InferenceService [isvc-logger-raw-c27ac] is Ready metadata: creationTimestamp: "2026-07-07T15:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:14:01Z" name: isvc-logger-raw-c27ac.18c00ab8c3259626 namespace: kserve-ci-e2e-test resourceVersion: "17988" uid: bcbadb6a-da02-4500-9379-6586423ad79e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:17:08.306041Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-d3e813-predictor-5ccd495944-jplx5 namespace: kserve-ci-e2e-test resourceVersion: "19462" uid: a49682b8-c9c8-4d9f-a25d-b93afb1701ff kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-d3e813-predictor-5ccd495944-jplx5 to ip-10-0-143-168.ec2.internal metadata: creationTimestamp: "2026-07-07T15:17:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:17:08Z" name: isvc-primary-d3e813-predictor-5ccd495944-jplx5.18c00ae45951861b namespace: kserve-ci-e2e-test resourceVersion: "19468" uid: dc970a2d-ff99-4fdc-a4ff-38b39beaafe4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-77985f99fc-vc8tw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:08Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-d3e813-predictor-5ccd495944-jplx5 namespace: kserve-ci-e2e-test resourceVersion: "19474" uid: a49682b8-c9c8-4d9f-a25d-b93afb1701ff kind: Event lastTimestamp: "2026-07-07T15:17:08Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:17:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:17:08Z" name: isvc-primary-d3e813-predictor-5ccd495944-jplx5.18c00ae4734c10d2 namespace: kserve-ci-e2e-test resourceVersion: "19485" uid: ac02baf1-313d-4031-be89-078126baed8f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-d3e813-predictor-5ccd495944-jplx5 namespace: kserve-ci-e2e-test resourceVersion: "19466" uid: a49682b8-c9c8-4d9f-a25d-b93afb1701ff kind: Event lastTimestamp: "2026-07-07T15:17:08Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:09d6f486359107a6e5d5aadd7d4fa2ea151504e2bf98f31f8912a941eb631c4b" already present on machine metadata: creationTimestamp: "2026-07-07T15:17:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:08Z" name: isvc-primary-d3e813-predictor-5ccd495944-jplx5.18c00ae474ccd9c7 namespace: kserve-ci-e2e-test resourceVersion: "19487" uid: 467459f5-8533-4769-b9a4-e699c810d132 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-d3e813-predictor-5ccd495944-jplx5 namespace: kserve-ci-e2e-test resourceVersion: "19466" uid: a49682b8-c9c8-4d9f-a25d-b93afb1701ff kind: Event lastTimestamp: "2026-07-07T15:17:08Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T15:17:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:08Z" name: isvc-primary-d3e813-predictor-5ccd495944-jplx5.18c00ae478292fc3 namespace: kserve-ci-e2e-test resourceVersion: "19488" uid: c9afdc47-a5ef-4608-a23b-d306f92ab4f0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-d3e813-predictor-5ccd495944-jplx5 namespace: kserve-ci-e2e-test resourceVersion: "19466" uid: a49682b8-c9c8-4d9f-a25d-b93afb1701ff kind: Event lastTimestamp: "2026-07-07T15:17:08Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T15:17:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:08Z" name: isvc-primary-d3e813-predictor-5ccd495944-jplx5.18c00ae4789d316b namespace: kserve-ci-e2e-test resourceVersion: "19489" uid: 3b4cd2c2-f146-49cd-934e-35a7c14997aa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-d3e813-predictor-5ccd495944-jplx5 namespace: kserve-ci-e2e-test resourceVersion: "19466" uid: a49682b8-c9c8-4d9f-a25d-b93afb1701ff kind: Event lastTimestamp: "2026-07-07T15:17:13Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:17:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:13Z" name: isvc-primary-d3e813-predictor-5ccd495944-jplx5.18c00ae57bb64cfb namespace: kserve-ci-e2e-test resourceVersion: "19527" uid: 0029ecbc-2bdf-4226-b3bc-910fcdd4da27 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-d3e813-predictor-5ccd495944-jplx5 namespace: kserve-ci-e2e-test resourceVersion: "19466" uid: a49682b8-c9c8-4d9f-a25d-b93afb1701ff kind: Event lastTimestamp: "2026-07-07T15:17:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:17:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:13Z" name: isvc-primary-d3e813-predictor-5ccd495944-jplx5.18c00ae57f9033f8 namespace: kserve-ci-e2e-test resourceVersion: "19531" uid: 6cfa58d2-b79d-48f6-b7de-78d589e2e5d2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-d3e813-predictor-5ccd495944-jplx5 namespace: kserve-ci-e2e-test resourceVersion: "19466" uid: a49682b8-c9c8-4d9f-a25d-b93afb1701ff kind: Event lastTimestamp: "2026-07-07T15:17:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:17:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:13Z" name: isvc-primary-d3e813-predictor-5ccd495944-jplx5.18c00ae5800614c7 namespace: kserve-ci-e2e-test resourceVersion: "19533" uid: b5f12d6c-8a97-497e-874b-6fba8a71b2d3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-d3e813-predictor-5ccd495944-jplx5 namespace: kserve-ci-e2e-test resourceVersion: "19466" uid: a49682b8-c9c8-4d9f-a25d-b93afb1701ff kind: Event lastTimestamp: "2026-07-07T15:17:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:17:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:13Z" name: isvc-primary-d3e813-predictor-5ccd495944-jplx5.18c00ae580108956 namespace: kserve-ci-e2e-test resourceVersion: "19534" uid: daec7233-60c0-4946-ad6e-09ce3bbb5eeb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-d3e813-predictor-5ccd495944-jplx5 namespace: kserve-ci-e2e-test resourceVersion: "19466" uid: a49682b8-c9c8-4d9f-a25d-b93afb1701ff kind: Event lastTimestamp: "2026-07-07T15:17:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:17:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:13Z" name: isvc-primary-d3e813-predictor-5ccd495944-jplx5.18c00ae58421c9b9 namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: fa093fde-38d7-4381-b65e-7fd66eea29fa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-d3e813-predictor-5ccd495944-jplx5 namespace: kserve-ci-e2e-test resourceVersion: "19466" uid: a49682b8-c9c8-4d9f-a25d-b93afb1701ff kind: Event lastTimestamp: "2026-07-07T15:17:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:17:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:13Z" name: isvc-primary-d3e813-predictor-5ccd495944-jplx5.18c00ae5849632ed namespace: kserve-ci-e2e-test resourceVersion: "19538" uid: b1787a2c-a212-4f17-98e4-2534d5159edf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T15:17:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-d3e813-predictor-5ccd495944-jplx5 namespace: kserve-ci-e2e-test resourceVersion: "19466" uid: a49682b8-c9c8-4d9f-a25d-b93afb1701ff kind: Event lastTimestamp: "2026-07-07T15:18:59Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:17:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:18:59Z" name: isvc-primary-d3e813-predictor-5ccd495944-jplx5.18c00ae5f3eb06b1 namespace: kserve-ci-e2e-test resourceVersion: "20407" uid: a736983c-d1d0-4ab7-a2ce-4fe139878ceb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-d3e813-predictor-5ccd495944-jplx5 namespace: kserve-ci-e2e-test resourceVersion: "19466" uid: a49682b8-c9c8-4d9f-a25d-b93afb1701ff kind: Event lastTimestamp: "2026-07-07T15:18:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:18:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:18:56Z" name: isvc-primary-d3e813-predictor-5ccd495944-jplx5.18c00afd92f8b70a namespace: kserve-ci-e2e-test resourceVersion: "20374" uid: ae6b4abd-1905-46ee-89e5-d236b47d5f12 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-d3e813-predictor-5ccd495944-jplx5 namespace: kserve-ci-e2e-test resourceVersion: "19466" uid: a49682b8-c9c8-4d9f-a25d-b93afb1701ff kind: Event lastTimestamp: "2026-07-07T15:18:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:18:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:18:56Z" name: isvc-primary-d3e813-predictor-5ccd495944-jplx5.18c00afd92f93e30 namespace: kserve-ci-e2e-test resourceVersion: "20375" uid: 57244515-2ab3-456f-b25e-4b5cc6b2dff4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-d3e813-predictor-5ccd495944-jplx5 namespace: kserve-ci-e2e-test resourceVersion: "19466" uid: a49682b8-c9c8-4d9f-a25d-b93afb1701ff kind: Event lastTimestamp: "2026-07-07T15:19:01Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:19:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:19:01Z" name: isvc-primary-d3e813-predictor-5ccd495944-jplx5.18c00afea239b315 namespace: kserve-ci-e2e-test resourceVersion: "20419" uid: 154f2690-48a2-4c0d-8d0d-b01830b48d71 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-d3e813-predictor-5ccd495944 namespace: kserve-ci-e2e-test resourceVersion: "19453" uid: 113daced-23c9-4d2b-b666-4845d81026dc kind: Event lastTimestamp: "2026-07-07T15:17:08Z" message: 'Created pod: isvc-primary-d3e813-predictor-5ccd495944-jplx5' metadata: creationTimestamp: "2026-07-07T15:17:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:08Z" name: isvc-primary-d3e813-predictor-5ccd495944.18c00ae458e133b3 namespace: kserve-ci-e2e-test resourceVersion: "19465" uid: 75951703-1522-4506-9636-7ecb99e604d0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-d3e813-predictor namespace: kserve-ci-e2e-test resourceVersion: "19451" uid: 18badac6-185c-4bbb-9d57-bdf329155abf kind: Event lastTimestamp: "2026-07-07T15:17:08Z" message: Scaled up replica set isvc-primary-d3e813-predictor-5ccd495944 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:17:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:08Z" name: isvc-primary-d3e813-predictor.18c00ae4547f099b namespace: kserve-ci-e2e-test resourceVersion: "19455" uid: 004881e0-2b89-46d0-8495-45e486007b48 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:17:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-d3e813-predictor namespace: kserve-ci-e2e-test resourceVersion: "19452" uid: 45976ea6-f60c-4251-9b7b-3d72d959aacb kind: Event lastTimestamp: "2026-07-07T15:17:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:17:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:38Z" name: isvc-primary-d3e813-predictor.18c00ae7d3869a09 namespace: kserve-ci-e2e-test resourceVersion: "19713" uid: 71496ec7-08b7-4113-b636-905a209eda67 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:17:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-d3e813-predictor namespace: kserve-ci-e2e-test resourceVersion: "19452" uid: 45976ea6-f60c-4251-9b7b-3d72d959aacb kind: Event lastTimestamp: "2026-07-07T15:17:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:17:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:17:38Z" name: isvc-primary-d3e813-predictor.18c00ae7d3f57a3b namespace: kserve-ci-e2e-test resourceVersion: "19715" uid: 089211e9-ebc2-43c0-8324-744b96f82036 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:17:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-d3e813-predictor namespace: kserve-ci-e2e-test resourceVersion: "19615" uid: 45976ea6-f60c-4251-9b7b-3d72d959aacb kind: Event lastTimestamp: "2026-07-07T15:18:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:17:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:18:53Z" name: isvc-primary-d3e813-predictor.18c00aeed3162260 namespace: kserve-ci-e2e-test resourceVersion: "20317" uid: bae981a5-b840-4e0d-b0bb-e3fe518b8ec3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:17:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-d3e813-predictor namespace: kserve-ci-e2e-test resourceVersion: "19615" uid: 45976ea6-f60c-4251-9b7b-3d72d959aacb kind: Event lastTimestamp: "2026-07-07T15:18:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:17:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:18:53Z" name: isvc-primary-d3e813-predictor.18c00aeed391fa4e namespace: kserve-ci-e2e-test resourceVersion: "20318" uid: 9be9ab9e-73f1-4241-bde4-32e044bb4959 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-07T15:17:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-d3e813 namespace: kserve-ci-e2e-test resourceVersion: "19444" uid: 44094b72-678f-4e83-ad79-2a97d542ed5f kind: Event lastTimestamp: "2026-07-07T15:18:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:17:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:18:30Z" name: isvc-primary-d3e813.18c00ae4547a55a0 namespace: kserve-ci-e2e-test resourceVersion: "20088" uid: 48f4ecdf-96d4-48e5-b4d4-c5b2150e1770 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:17:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-d3e813 namespace: kserve-ci-e2e-test resourceVersion: "19444" uid: 44094b72-678f-4e83-ad79-2a97d542ed5f kind: Event lastTimestamp: "2026-07-07T15:17:08Z" message: 'Failed to update status for InferenceService "isvc-primary-d3e813": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-d3e813": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:17:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:17:08Z" name: isvc-primary-d3e813.18c00ae455196075 namespace: kserve-ci-e2e-test resourceVersion: "19476" uid: c7d70bd7-14d4-4c82-a385-782eaa4d2861 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:17:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-d3e813 namespace: kserve-ci-e2e-test resourceVersion: "19444" uid: 44094b72-678f-4e83-ad79-2a97d542ed5f kind: Event lastTimestamp: "2026-07-07T15:17:08Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-d3e813": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:17:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:17:08Z" name: isvc-primary-d3e813.18c00ae45519a705 namespace: kserve-ci-e2e-test resourceVersion: "19477" uid: d44ae54f-01bf-4a0a-99ad-7bfaceb325e8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-d3e813 namespace: kserve-ci-e2e-test resourceVersion: "20086" uid: 44094b72-678f-4e83-ad79-2a97d542ed5f kind: Event lastTimestamp: "2026-07-07T15:18:30Z" message: InferenceService [isvc-primary-d3e813] is Ready metadata: creationTimestamp: "2026-07-07T15:18:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:18:30Z" name: isvc-primary-d3e813.18c00af75f90ce7d namespace: kserve-ci-e2e-test resourceVersion: "20087" uid: f16388e7-f39f-4193-bc7e-be0df8a1a313 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:06:52.069166Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9 namespace: kserve-ci-e2e-test resourceVersion: "14189" uid: 91eec569-d4d4-406f-8927-f67a43479ae6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9 to ip-10-0-143-168.ec2.internal metadata: creationTimestamp: "2026-07-07T15:06:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:06:52Z" name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9.18c00a54debcf6a3 namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: 4a2c88c9-fb8b-4bf6-98ca-0a66030d2a53 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-77985f99fc-vc8tw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9 namespace: kserve-ci-e2e-test resourceVersion: "14196" uid: 91eec569-d4d4-406f-8927-f67a43479ae6 kind: Event lastTimestamp: "2026-07-07T15:06:52Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:06:52Z" name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9.18c00a54f8671f59 namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: 329cca9f-cbe3-4b02-ba93-4921af80aaae reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9 namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 91eec569-d4d4-406f-8927-f67a43479ae6 kind: Event lastTimestamp: "2026-07-07T15:06:52Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:09d6f486359107a6e5d5aadd7d4fa2ea151504e2bf98f31f8912a941eb631c4b" already present on machine metadata: creationTimestamp: "2026-07-07T15:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:52Z" name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9.18c00a54fa2000a1 namespace: kserve-ci-e2e-test resourceVersion: "14210" uid: a2624054-e1f4-4f60-8d1b-141e2f454684 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9 namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 91eec569-d4d4-406f-8927-f67a43479ae6 kind: Event lastTimestamp: "2026-07-07T15:06:52Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T15:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:52Z" name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9.18c00a54fd4f4360 namespace: kserve-ci-e2e-test resourceVersion: "14211" uid: c1c83645-708e-4aec-8f90-7a01dd236657 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9 namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 91eec569-d4d4-406f-8927-f67a43479ae6 kind: Event lastTimestamp: "2026-07-07T15:06:52Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T15:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:52Z" name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9.18c00a54fdba5a2e namespace: kserve-ci-e2e-test resourceVersion: "14212" uid: acd3402b-7695-4cbb-a8cf-d50827210109 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9 namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 91eec569-d4d4-406f-8927-f67a43479ae6 kind: Event lastTimestamp: "2026-07-07T15:06:56Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" metadata: creationTimestamp: "2026-07-07T15:06:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:06:56Z" name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9.18c00a55dd06e845 namespace: kserve-ci-e2e-test resourceVersion: "14242" uid: 577d7d29-5300-49b7-a303-8ca1fbb8c399 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9 namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 91eec569-d4d4-406f-8927-f67a43479ae6 kind: Event lastTimestamp: "2026-07-07T15:07:09Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" in 13.229s (13.229s including waiting). Image size: 1555863462 bytes.' metadata: creationTimestamp: "2026-07-07T15:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:09Z" name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9.18c00a58f18ce1dd namespace: kserve-ci-e2e-test resourceVersion: "14335" uid: bacf1f6e-28e5-4a46-b2b9-a9d33e060ce6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9 namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 91eec569-d4d4-406f-8927-f67a43479ae6 kind: Event lastTimestamp: "2026-07-07T15:07:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:09Z" name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9.18c00a58f4d3fa9c namespace: kserve-ci-e2e-test resourceVersion: "14336" uid: 6cc17c9e-3dba-48bd-a3bb-c6e6fe4a6513 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9 namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 91eec569-d4d4-406f-8927-f67a43479ae6 kind: Event lastTimestamp: "2026-07-07T15:07:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:09Z" name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9.18c00a58f546b942 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: 446a8c0f-94fc-4150-a11a-eb9685232166 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9 namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 91eec569-d4d4-406f-8927-f67a43479ae6 kind: Event lastTimestamp: "2026-07-07T15:07:09Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-07T15:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:09Z" name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9.18c00a58f554827a namespace: kserve-ci-e2e-test resourceVersion: "14338" uid: 1f282ce0-a843-4fd3-9cc6-eb970f583420 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9 namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 91eec569-d4d4-406f-8927-f67a43479ae6 kind: Event lastTimestamp: "2026-07-07T15:07:12Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.917s (2.917s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-07T15:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:12Z" name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9.18c00a59a337a6d4 namespace: kserve-ci-e2e-test resourceVersion: "14362" uid: eee240f5-37c1-43b4-a72d-ffb2310440b8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9 namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 91eec569-d4d4-406f-8927-f67a43479ae6 kind: Event lastTimestamp: "2026-07-07T15:07:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:12Z" name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9.18c00a59a6347bb5 namespace: kserve-ci-e2e-test resourceVersion: "14363" uid: 1c071176-1e2e-4785-b4d8-1bc250a968e7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9 namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 91eec569-d4d4-406f-8927-f67a43479ae6 kind: Event lastTimestamp: "2026-07-07T15:07:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:12Z" name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9.18c00a59a6aed039 namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 3f4c48be-b57f-4833-8941-24e0ce925912 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9 namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 91eec569-d4d4-406f-8927-f67a43479ae6 kind: Event lastTimestamp: "2026-07-07T15:07:12Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:c2ec3c7b4f4e1e45c65c5dc6bb443d6d9f93bbd95edfb128e24e40a56ed9bf66" metadata: creationTimestamp: "2026-07-07T15:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:12Z" name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9.18c00a59a6bdd6be namespace: kserve-ci-e2e-test resourceVersion: "14365" uid: c63360db-9d26-4d9e-bd90-dfae07ee51d8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9 namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 91eec569-d4d4-406f-8927-f67a43479ae6 kind: Event lastTimestamp: "2026-07-07T15:07:15Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:c2ec3c7b4f4e1e45c65c5dc6bb443d6d9f93bbd95edfb128e24e40a56ed9bf66" in 3.235s (3.235s including waiting). Image size: 239995486 bytes.' metadata: creationTimestamp: "2026-07-07T15:07:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:15Z" name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9.18c00a5a679f6efb namespace: kserve-ci-e2e-test resourceVersion: "14386" uid: b027dc00-af3f-4983-879e-47c5485bfeb5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9 namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 91eec569-d4d4-406f-8927-f67a43479ae6 kind: Event lastTimestamp: "2026-07-07T15:07:15Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-07T15:07:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:15Z" name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9.18c00a5a6b7341e8 namespace: kserve-ci-e2e-test resourceVersion: "14389" uid: bc320cd3-6ab4-4cbb-969b-e1e7350acb56 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:07:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9 namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 91eec569-d4d4-406f-8927-f67a43479ae6 kind: Event lastTimestamp: "2026-07-07T15:07:15Z" message: Started container agent metadata: creationTimestamp: "2026-07-07T15:07:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:07:15Z" name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9.18c00a5a6be7a772 namespace: kserve-ci-e2e-test resourceVersion: "14390" uid: 00610166-3d47-420c-b550-097690a2d478 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-07T15:07:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9 namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 91eec569-d4d4-406f-8927-f67a43479ae6 kind: Event lastTimestamp: "2026-07-07T15:08:49Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:07:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:49Z" name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9.18c00a5a8bdfd5a2 namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: b3a77717-8153-4634-b743-2f50ead820e9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T15:07:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9 namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 91eec569-d4d4-406f-8927-f67a43479ae6 kind: Event lastTimestamp: "2026-07-07T15:08:39Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T15:07:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:39Z" name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9.18c00a5ac7b14751 namespace: kserve-ci-e2e-test resourceVersion: "15126" uid: c41bb077-492f-4075-8df4-43f7f8d0c1c6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9 namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 91eec569-d4d4-406f-8927-f67a43479ae6 kind: Event lastTimestamp: "2026-07-07T15:08:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:37Z" name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9.18c00a6d55a4bb53 namespace: kserve-ci-e2e-test resourceVersion: "15046" uid: 2215acad-a651-41f1-b58e-fdf07b409ae9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9 namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 91eec569-d4d4-406f-8927-f67a43479ae6 kind: Event lastTimestamp: "2026-07-07T15:08:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:37Z" name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9.18c00a6d55a5266a namespace: kserve-ci-e2e-test resourceVersion: "15052" uid: 60ddd984-02fa-448e-90e7-bcabd8c3039d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9 namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 91eec569-d4d4-406f-8927-f67a43479ae6 kind: Event lastTimestamp: "2026-07-07T15:08:37Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-07T15:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:37Z" name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9.18c00a6d55aa2f06 namespace: kserve-ci-e2e-test resourceVersion: "15055" uid: f189f64c-acac-4bc9-814a-faecc87bdc90 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:08:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9 namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 91eec569-d4d4-406f-8927-f67a43479ae6 kind: Event lastTimestamp: "2026-07-07T15:08:47Z" message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": dial tcp 10.134.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:47Z" name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9.18c00a6d680b0d16 namespace: kserve-ci-e2e-test resourceVersion: "15195" uid: 851496bf-dc89-4e5c-9686-c71823b64a97 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c namespace: kserve-ci-e2e-test resourceVersion: "14172" uid: 39c4a995-636b-40c3-87fe-3e7e75dd418f kind: Event lastTimestamp: "2026-07-07T15:06:52Z" message: 'Created pod: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c-jjtx9' metadata: creationTimestamp: "2026-07-07T15:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:06:52Z" name: isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c.18c00a54dd983e2c namespace: kserve-ci-e2e-test resourceVersion: "14190" uid: 8a73def0-afae-46a9-9660-c756e1c83efd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:06:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-df08e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14170" uid: 4ccf236c-165c-49aa-876d-319fe1e9b0b8 kind: Event lastTimestamp: "2026-07-07T15:06:51Z" message: Scaled up replica set isvc-raw-sklearn-batcher-df08e-predictor-6c5fdf897c from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:06:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:06:51Z" name: isvc-raw-sklearn-batcher-df08e-predictor.18c00a54d2316e87 namespace: kserve-ci-e2e-test resourceVersion: "14174" uid: 23e65e08-04a1-472e-8cab-6830c2e33851 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:07:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-df08e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14181" uid: cee80a02-c429-482f-9f4b-647f96853b8f kind: Event lastTimestamp: "2026-07-07T15:07:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:07:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:07:37Z" name: isvc-raw-sklearn-batcher-df08e-predictor.18c00a585760fc20 namespace: kserve-ci-e2e-test resourceVersion: "14548" uid: 11d409fb-3f2f-4032-b440-6d0fcc86ca10 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:07:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-df08e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14181" uid: cee80a02-c429-482f-9f4b-647f96853b8f kind: Event lastTimestamp: "2026-07-07T15:07:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:07:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:07:37Z" name: isvc-raw-sklearn-batcher-df08e-predictor.18c00a5857dba0e7 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: 8d8f8f4f-a0e8-4e17-acdd-74e9809860d3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T15:07:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-df08e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: cee80a02-c429-482f-9f4b-647f96853b8f kind: Event lastTimestamp: "2026-07-07T15:08:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:08:37Z" name: isvc-raw-sklearn-batcher-df08e-predictor.18c00a62d50d0d57 namespace: kserve-ci-e2e-test resourceVersion: "15024" uid: d2762438-34d5-490e-810c-3d74477dcce7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T15:07:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-df08e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: cee80a02-c429-482f-9f4b-647f96853b8f kind: Event lastTimestamp: "2026-07-07T15:08:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:08:37Z" name: isvc-raw-sklearn-batcher-df08e-predictor.18c00a62d5875560 namespace: kserve-ci-e2e-test resourceVersion: "15030" uid: 4504e982-e44b-4456-80cd-553f15e08196 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T15:06:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-df08e namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 0b4ca42d-92f6-475f-a925-52e62f057823 kind: Event lastTimestamp: "2026-07-07T15:08:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:06:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:08:29Z" name: isvc-raw-sklearn-batcher-df08e.18c00a54d7eb4fc7 namespace: kserve-ci-e2e-test resourceVersion: "14935" uid: cee100f7-1296-4832-ad46-6e843c3d06a7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:06:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-df08e namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 0b4ca42d-92f6-475f-a925-52e62f057823 kind: Event lastTimestamp: "2026-07-07T15:06:52Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-df08e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-df08e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:06:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:06:52Z" name: isvc-raw-sklearn-batcher-df08e.18c00a54d886b0ac namespace: kserve-ci-e2e-test resourceVersion: "14198" uid: cfe2ce4b-09cb-4a50-9f18-2fa921fc9865 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:06:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-df08e namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 0b4ca42d-92f6-475f-a925-52e62f057823 kind: Event lastTimestamp: "2026-07-07T15:06:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-df08e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:06:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:06:52Z" name: isvc-raw-sklearn-batcher-df08e.18c00a54d8870579 namespace: kserve-ci-e2e-test resourceVersion: "14201" uid: 65fd70db-ae68-41b9-87db-4a1aa90ca9b9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-df08e namespace: kserve-ci-e2e-test resourceVersion: "14933" uid: 0b4ca42d-92f6-475f-a925-52e62f057823 kind: Event lastTimestamp: "2026-07-07T15:08:29Z" message: InferenceService [isvc-raw-sklearn-batcher-df08e] is Ready metadata: creationTimestamp: "2026-07-07T15:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:08:29Z" name: isvc-raw-sklearn-batcher-df08e.18c00a6b930ad3f0 namespace: kserve-ci-e2e-test resourceVersion: "14934" uid: b454f8a6-ae52-4974-9626-535bebff40c4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:18:38.564593Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-d3e813-predictor-64cc95f8c-zbfw8 namespace: kserve-ci-e2e-test resourceVersion: "20180" uid: 6a27b72f-4351-45a7-8978-2770feeb7d7f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-d3e813-predictor-64cc95f8c-zbfw8 to ip-10-0-143-168.ec2.internal metadata: creationTimestamp: "2026-07-07T15:18:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:18:38Z" name: isvc-secondary-d3e813-predictor-64cc95f8c-zbfw8.18c00af95d25b336 namespace: kserve-ci-e2e-test resourceVersion: "20184" uid: 057f375f-2bb8-403b-956e-f1d7b7a6faa4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-77985f99fc-vc8tw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:38Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-d3e813-predictor-64cc95f8c-zbfw8 namespace: kserve-ci-e2e-test resourceVersion: "20181" uid: 6a27b72f-4351-45a7-8978-2770feeb7d7f kind: Event lastTimestamp: "2026-07-07T15:18:38Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-secondary-d3e813-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T15:18:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:18:38Z" name: isvc-secondary-d3e813-predictor-64cc95f8c-zbfw8.18c00af96e428b98 namespace: kserve-ci-e2e-test resourceVersion: "20196" uid: ac6e7ddd-81a6-430e-8fd9-81eebbc70ae9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-d3e813-predictor-64cc95f8c-zbfw8 namespace: kserve-ci-e2e-test resourceVersion: "20189" uid: 6a27b72f-4351-45a7-8978-2770feeb7d7f kind: Event lastTimestamp: "2026-07-07T15:18:39Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:18:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:18:39Z" name: isvc-secondary-d3e813-predictor-64cc95f8c-zbfw8.18c00af99aaecbfb namespace: kserve-ci-e2e-test resourceVersion: "20203" uid: 686215a1-b429-4308-ada0-17532f1660d8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:18:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-d3e813-predictor-64cc95f8c-zbfw8 namespace: kserve-ci-e2e-test resourceVersion: "20181" uid: 6a27b72f-4351-45a7-8978-2770feeb7d7f kind: Event lastTimestamp: "2026-07-07T15:18:44Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:09d6f486359107a6e5d5aadd7d4fa2ea151504e2bf98f31f8912a941eb631c4b" already present on machine metadata: creationTimestamp: "2026-07-07T15:18:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:18:44Z" name: isvc-secondary-d3e813-predictor-64cc95f8c-zbfw8.18c00af99c8229be namespace: kserve-ci-e2e-test resourceVersion: "20245" uid: da8c92d1-f445-4c37-bfad-56988b0229ab reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:18:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-d3e813-predictor-64cc95f8c-zbfw8 namespace: kserve-ci-e2e-test resourceVersion: "20181" uid: 6a27b72f-4351-45a7-8978-2770feeb7d7f kind: Event lastTimestamp: "2026-07-07T15:18:44Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T15:18:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:18:44Z" name: isvc-secondary-d3e813-predictor-64cc95f8c-zbfw8.18c00af99f641e57 namespace: kserve-ci-e2e-test resourceVersion: "20247" uid: 8332f38f-b91f-4a3b-b299-ada432846aa9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:18:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-d3e813-predictor-64cc95f8c-zbfw8 namespace: kserve-ci-e2e-test resourceVersion: "20181" uid: 6a27b72f-4351-45a7-8978-2770feeb7d7f kind: Event lastTimestamp: "2026-07-07T15:18:44Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T15:18:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:18:44Z" name: isvc-secondary-d3e813-predictor-64cc95f8c-zbfw8.18c00af99fd3b1dc namespace: kserve-ci-e2e-test resourceVersion: "20248" uid: 0b910047-e744-467c-b6d0-01b167997eb7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-d3e813-predictor-64cc95f8c-zbfw8 namespace: kserve-ci-e2e-test resourceVersion: "20181" uid: 6a27b72f-4351-45a7-8978-2770feeb7d7f kind: Event lastTimestamp: "2026-07-07T15:18:46Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-d3e813-predictor-64cc95f8c-zbfw8_kserve-ci-e2e-test(6a27b72f-4351-45a7-8978-2770feeb7d7f) metadata: creationTimestamp: "2026-07-07T15:18:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:18:46Z" name: isvc-secondary-d3e813-predictor-64cc95f8c-zbfw8.18c00afb3be106c4 namespace: kserve-ci-e2e-test resourceVersion: "20266" uid: 0966ae41-36bd-4ed9-8cb0-bd1fb779190d reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-d3e813-predictor-64cc95f8c namespace: kserve-ci-e2e-test resourceVersion: "20171" uid: e65cd972-6bd1-49b6-8d9c-14c81859e76f kind: Event lastTimestamp: "2026-07-07T15:18:38Z" message: 'Created pod: isvc-secondary-d3e813-predictor-64cc95f8c-zbfw8' metadata: creationTimestamp: "2026-07-07T15:18:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:18:38Z" name: isvc-secondary-d3e813-predictor-64cc95f8c.18c00af95cbbbbd8 namespace: kserve-ci-e2e-test resourceVersion: "20182" uid: 8b0c3c42-30ab-4fee-9b33-01a0f19496c9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-d3e813-predictor namespace: kserve-ci-e2e-test resourceVersion: "20168" uid: e5a374f4-732d-4360-9efe-690ae9787d40 kind: Event lastTimestamp: "2026-07-07T15:18:38Z" message: Scaled up replica set isvc-secondary-d3e813-predictor-64cc95f8c from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:18:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:18:38Z" name: isvc-secondary-d3e813-predictor.18c00af9561b11b4 namespace: kserve-ci-e2e-test resourceVersion: "20173" uid: 74977a5d-5cbd-4f6d-9486-6a8673fc554d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-d3e813-predictor namespace: kserve-ci-e2e-test resourceVersion: "20170" uid: 97b1da1c-c390-4794-b25c-89be4b1d0559 kind: Event lastTimestamp: "2026-07-07T15:18:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:18:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:18:53Z" name: isvc-secondary-d3e813-predictor.18c00afcd4ac1bcc namespace: kserve-ci-e2e-test resourceVersion: "20319" uid: 64980d82-1e5f-449b-963d-5e039130472a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-d3e813-predictor namespace: kserve-ci-e2e-test resourceVersion: "20170" uid: 97b1da1c-c390-4794-b25c-89be4b1d0559 kind: Event lastTimestamp: "2026-07-07T15:18:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:18:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:18:53Z" name: isvc-secondary-d3e813-predictor.18c00afcd5157103 namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: d2698a5f-5fe2-4dd1-8113-4ab8e2b1f338 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-07-07T15:18:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-d3e813 namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: 283c61f0-362a-42d9-bdcc-48dc4ab43f77 kind: Event lastTimestamp: "2026-07-07T15:18:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:18:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:18:46Z" name: isvc-secondary-d3e813.18c00af9563260f2 namespace: kserve-ci-e2e-test resourceVersion: "20274" uid: 598205a8-6ad6-4629-a802-51ac2a9c8136 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-d3e813 namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: 283c61f0-362a-42d9-bdcc-48dc4ab43f77 kind: Event lastTimestamp: "2026-07-07T15:18:38Z" message: 'Failed to update status for InferenceService "isvc-secondary-d3e813": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-d3e813": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:18:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:18:38Z" name: isvc-secondary-d3e813.18c00af95750a6ce namespace: kserve-ci-e2e-test resourceVersion: "20175" uid: a0101c1a-82c8-477a-bd84-11f638d62648 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:18:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-d3e813 namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: 283c61f0-362a-42d9-bdcc-48dc4ab43f77 kind: Event lastTimestamp: "2026-07-07T15:18:38Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-d3e813": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:18:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:18:38Z" name: isvc-secondary-d3e813.18c00af95750e702 namespace: kserve-ci-e2e-test resourceVersion: "20176" uid: f62d4edd-4906-4dbc-96b1-6d89354a9041 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:08:37.144343Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: a71558c8-c93c-4383-9db1-041926890e0b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz to ip-10-0-143-168.ec2.internal metadata: creationTimestamp: "2026-07-07T15:08:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:08:37Z" name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz.18c00a6d55b4e9d6 namespace: kserve-ci-e2e-test resourceVersion: "15048" uid: 8bd7eda6-5c35-4c57-9db6-ba9d8fabc232 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-77985f99fc-vc8tw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:37Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: a71558c8-c93c-4383-9db1-041926890e0b kind: Event lastTimestamp: "2026-07-07T15:08:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-raw-e3a4a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T15:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:37Z" name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz.18c00a6d623dc581 namespace: kserve-ci-e2e-test resourceVersion: "15084" uid: 2c3a5f29-b5e8-4458-836a-5d867b44e83e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:38Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz namespace: kserve-ci-e2e-test resourceVersion: "15060" uid: a71558c8-c93c-4383-9db1-041926890e0b kind: Event lastTimestamp: "2026-07-07T15:08:38Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:08:38Z" name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz.18c00a6d949aa727 namespace: kserve-ci-e2e-test resourceVersion: "15108" uid: 8c125fde-2a53-4105-9069-018cfc512fa0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: a71558c8-c93c-4383-9db1-041926890e0b kind: Event lastTimestamp: "2026-07-07T15:08:38Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:09d6f486359107a6e5d5aadd7d4fa2ea151504e2bf98f31f8912a941eb631c4b" already present on machine metadata: creationTimestamp: "2026-07-07T15:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:38Z" name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz.18c00a6d96e3a330 namespace: kserve-ci-e2e-test resourceVersion: "15110" uid: 55dffb54-fe0a-4801-aba8-80eed70fe4e4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: a71558c8-c93c-4383-9db1-041926890e0b kind: Event lastTimestamp: "2026-07-07T15:08:38Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T15:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:38Z" name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz.18c00a6d9a3bf4b4 namespace: kserve-ci-e2e-test resourceVersion: "15111" uid: ea910df7-4786-4569-93d5-bd92967b80d5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: a71558c8-c93c-4383-9db1-041926890e0b kind: Event lastTimestamp: "2026-07-07T15:08:38Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T15:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:38Z" name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz.18c00a6d9ab535dc namespace: kserve-ci-e2e-test resourceVersion: "15113" uid: e76ccd31-5f62-4b31-8efc-fbdb6936ed64 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: a71558c8-c93c-4383-9db1-041926890e0b kind: Event lastTimestamp: "2026-07-07T15:08:42Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:08:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:42Z" name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz.18c00a6ea8ffecd4 namespace: kserve-ci-e2e-test resourceVersion: "15157" uid: 61e1f8de-0ff9-4675-bb1a-b1b0ded0724f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: a71558c8-c93c-4383-9db1-041926890e0b kind: Event lastTimestamp: "2026-07-07T15:08:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:08:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:42Z" name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz.18c00a6ead30196d namespace: kserve-ci-e2e-test resourceVersion: "15161" uid: 58bd268e-6505-45d5-b3c8-4eb4623c181e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: a71558c8-c93c-4383-9db1-041926890e0b kind: Event lastTimestamp: "2026-07-07T15:08:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:08:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:42Z" name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz.18c00a6eadcb7e35 namespace: kserve-ci-e2e-test resourceVersion: "15162" uid: 5d727545-6c64-40b3-a4c1-c440f3910121 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: a71558c8-c93c-4383-9db1-041926890e0b kind: Event lastTimestamp: "2026-07-07T15:08:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:08:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:42Z" name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz.18c00a6eadd8f7eb namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: 64a5bff9-90f2-408a-ab11-0e40a3197a34 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: a71558c8-c93c-4383-9db1-041926890e0b kind: Event lastTimestamp: "2026-07-07T15:08:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:08:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:42Z" name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz.18c00a6eb1f0c77b namespace: kserve-ci-e2e-test resourceVersion: "15165" uid: cb119547-d7f3-4b52-a50d-1374aafa31e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: a71558c8-c93c-4383-9db1-041926890e0b kind: Event lastTimestamp: "2026-07-07T15:08:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:08:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:42Z" name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz.18c00a6eb27e9dd9 namespace: kserve-ci-e2e-test resourceVersion: "15166" uid: 40efcf03-f223-42c2-96a6-220494ddc1bd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-07T15:08:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: a71558c8-c93c-4383-9db1-041926890e0b kind: Event lastTimestamp: "2026-07-07T15:10:40Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:08:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:40Z" name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz.18c00a6ee583a5ef namespace: kserve-ci-e2e-test resourceVersion: "16249" uid: 16d1b78c-87f0-49d7-87a9-231098e62275 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: a71558c8-c93c-4383-9db1-041926890e0b kind: Event lastTimestamp: "2026-07-07T15:10:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:10:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:37Z" name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz.18c00a8958d63528 namespace: kserve-ci-e2e-test resourceVersion: "16139" uid: 84441114-c98d-4979-b5e6-692155059e9b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: a71558c8-c93c-4383-9db1-041926890e0b kind: Event lastTimestamp: "2026-07-07T15:10:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:10:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:37Z" name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz.18c00a8958d9952a namespace: kserve-ci-e2e-test resourceVersion: "16144" uid: 6af87356-7399-4aa5-b4f7-2fec04e5987a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: a71558c8-c93c-4383-9db1-041926890e0b kind: Event lastTimestamp: "2026-07-07T15:10:40Z" message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp 10.134.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:10:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:40Z" name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz.18c00a8a2407de60 namespace: kserve-ci-e2e-test resourceVersion: "16247" uid: 35f23c22-904f-4ce1-8e76-eaea4dacae28 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749 namespace: kserve-ci-e2e-test resourceVersion: "15006" uid: 2ef5abe6-bdc6-489b-9fae-bbc24d99a678 kind: Event lastTimestamp: "2026-07-07T15:08:37Z" message: 'Created pod: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749-65zpz' metadata: creationTimestamp: "2026-07-07T15:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:08:37Z" name: isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749.18c00a6d54ff7677 namespace: kserve-ci-e2e-test resourceVersion: "15040" uid: cb11cdcd-dd89-4b8f-aa88-218ca192b5a8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-e3a4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15004" uid: a9ea87b1-3f50-4bdf-a91c-ec72968b7377 kind: Event lastTimestamp: "2026-07-07T15:08:37Z" message: Scaled up replica set isvc-sklearn-graph-raw-e3a4a-predictor-6f4c9d8749 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:08:37Z" name: isvc-sklearn-graph-raw-e3a4a-predictor.18c00a6d4f1dad3b namespace: kserve-ci-e2e-test resourceVersion: "15008" uid: cb183eb9-1558-4f25-b835-19ef2664c5a0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:08:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-e3a4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15005" uid: 58f5553a-83e3-46e7-b532-bc31111991d9 kind: Event lastTimestamp: "2026-07-07T15:09:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:09:07Z" name: isvc-sklearn-graph-raw-e3a4a-predictor.18c00a70cdb7bfb6 namespace: kserve-ci-e2e-test resourceVersion: "15356" uid: f1499ba9-07a6-4ee5-8041-38c0f14b250f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:08:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-e3a4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15005" uid: 58f5553a-83e3-46e7-b532-bc31111991d9 kind: Event lastTimestamp: "2026-07-07T15:09:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:09:07Z" name: isvc-sklearn-graph-raw-e3a4a-predictor.18c00a70ce33694f namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 4fd1e7b7-6cbd-4df9-b1bc-280581f215b8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:09:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-e3a4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15235" uid: 58f5553a-83e3-46e7-b532-bc31111991d9 kind: Event lastTimestamp: "2026-07-07T15:10:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:10:37Z" name: isvc-sklearn-graph-raw-e3a4a-predictor.18c00a77cc68ab34 namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: 162af202-830f-4f4a-b262-9e747eac8e49 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:09:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-e3a4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15235" uid: 58f5553a-83e3-46e7-b532-bc31111991d9 kind: Event lastTimestamp: "2026-07-07T15:10:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:10:37Z" name: isvc-sklearn-graph-raw-e3a4a-predictor.18c00a77cd078ac1 namespace: kserve-ci-e2e-test resourceVersion: "16080" uid: 80056a93-9606-4f70-9de5-8bd3e3b1f68a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T15:08:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-e3a4a namespace: kserve-ci-e2e-test resourceVersion: "14995" uid: 564e394f-4444-4c59-b4b9-21e6a8c18104 kind: Event lastTimestamp: "2026-07-07T15:10:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:10:11Z" name: isvc-sklearn-graph-raw-e3a4a.18c00a6d4f2eb0b7 namespace: kserve-ci-e2e-test resourceVersion: "15840" uid: 529cce6f-8f26-4563-8b93-3438bf5b97a4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-e3a4a namespace: kserve-ci-e2e-test resourceVersion: "14995" uid: 564e394f-4444-4c59-b4b9-21e6a8c18104 kind: Event lastTimestamp: "2026-07-07T15:08:37Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-e3a4a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-e3a4a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:08:37Z" name: isvc-sklearn-graph-raw-e3a4a.18c00a6d502da0ff namespace: kserve-ci-e2e-test resourceVersion: "15011" uid: 687ad4b2-8e05-41b8-9369-29d6c7e854f2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-e3a4a namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 564e394f-4444-4c59-b4b9-21e6a8c18104 kind: Event lastTimestamp: "2026-07-07T15:10:11Z" message: InferenceService [isvc-sklearn-graph-raw-e3a4a] is Ready metadata: creationTimestamp: "2026-07-07T15:10:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:10:11Z" name: isvc-sklearn-graph-raw-e3a4a.18c00a8331de0a72 namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: 89cc949c-c60c-4b57-9f57-8a0135784e8e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:10:37.457592Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl namespace: kserve-ci-e2e-test resourceVersion: "16128" uid: 3cd44159-004a-40b3-9a1e-90596e648f90 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl to ip-10-0-143-168.ec2.internal metadata: creationTimestamp: "2026-07-07T15:10:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:10:37Z" name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl.18c00a8958ef761b namespace: kserve-ci-e2e-test resourceVersion: "16138" uid: 4a6a9c28-6840-4e3c-a335-1436bbf263d6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-77985f99fc-vc8tw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:37Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl namespace: kserve-ci-e2e-test resourceVersion: "16152" uid: 3cd44159-004a-40b3-9a1e-90596e648f90 kind: Event lastTimestamp: "2026-07-07T15:10:37Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:10:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:10:37Z" name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl.18c00a89734f241e namespace: kserve-ci-e2e-test resourceVersion: "16204" uid: ab9aeef4-c595-4971-b59e-1776790f793f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: 3cd44159-004a-40b3-9a1e-90596e648f90 kind: Event lastTimestamp: "2026-07-07T15:10:37Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:09d6f486359107a6e5d5aadd7d4fa2ea151504e2bf98f31f8912a941eb631c4b" already present on machine metadata: creationTimestamp: "2026-07-07T15:10:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:37Z" name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl.18c00a8974ec25a1 namespace: kserve-ci-e2e-test resourceVersion: "16206" uid: 50f76f53-c6c5-4eaf-b12a-9152ba77ce47 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: 3cd44159-004a-40b3-9a1e-90596e648f90 kind: Event lastTimestamp: "2026-07-07T15:10:37Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T15:10:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:37Z" name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl.18c00a8977a068a6 namespace: kserve-ci-e2e-test resourceVersion: "16207" uid: 4fe8545b-1368-42e2-ac6e-eb055ad6bc33 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: 3cd44159-004a-40b3-9a1e-90596e648f90 kind: Event lastTimestamp: "2026-07-07T15:10:37Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T15:10:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:37Z" name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl.18c00a897814a2b5 namespace: kserve-ci-e2e-test resourceVersion: "16208" uid: a1261388-84c9-4891-ac83-b0acd19bcae5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: 3cd44159-004a-40b3-9a1e-90596e648f90 kind: Event lastTimestamp: "2026-07-07T15:10:42Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:10:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:42Z" name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl.18c00a8a8068d4ab namespace: kserve-ci-e2e-test resourceVersion: "16264" uid: fb312a0e-291f-4096-9f5f-8ba73cea6874 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: 3cd44159-004a-40b3-9a1e-90596e648f90 kind: Event lastTimestamp: "2026-07-07T15:10:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:10:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:42Z" name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl.18c00a8a83d1df5f namespace: kserve-ci-e2e-test resourceVersion: "16268" uid: ca090799-bfe5-4c9a-afc4-aebb4b57c872 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: 3cd44159-004a-40b3-9a1e-90596e648f90 kind: Event lastTimestamp: "2026-07-07T15:10:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:10:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:42Z" name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl.18c00a8a84356683 namespace: kserve-ci-e2e-test resourceVersion: "16271" uid: 08b1f070-cd83-4121-a2e5-a668207ef18f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: 3cd44159-004a-40b3-9a1e-90596e648f90 kind: Event lastTimestamp: "2026-07-07T15:10:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:10:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:42Z" name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl.18c00a8a843fc3bf namespace: kserve-ci-e2e-test resourceVersion: "16272" uid: 7f5d5372-1f41-4d13-ad8f-11e86d6840da reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: 3cd44159-004a-40b3-9a1e-90596e648f90 kind: Event lastTimestamp: "2026-07-07T15:10:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:10:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:42Z" name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl.18c00a8a8714add7 namespace: kserve-ci-e2e-test resourceVersion: "16275" uid: 6154ac85-f5b9-4a81-b77a-ba600836fe46 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: 3cd44159-004a-40b3-9a1e-90596e648f90 kind: Event lastTimestamp: "2026-07-07T15:10:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:10:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:42Z" name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl.18c00a8a87760d25 namespace: kserve-ci-e2e-test resourceVersion: "16276" uid: 1ba8d86a-07de-4144-b5c2-52f29279f322 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T15:10:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: 3cd44159-004a-40b3-9a1e-90596e648f90 kind: Event lastTimestamp: "2026-07-07T15:12:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:10:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:29Z" name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl.18c00a8abc7c8bc5 namespace: kserve-ci-e2e-test resourceVersion: "17228" uid: b4b9c6c8-8dda-4f92-8d7c-f87fa8f18a30 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: 3cd44159-004a-40b3-9a1e-90596e648f90 kind: Event lastTimestamp: "2026-07-07T15:12:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:12:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:27Z" name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl.18c00aa303ac112d namespace: kserve-ci-e2e-test resourceVersion: "17179" uid: 8d6d8e2e-a940-4b6c-9739-ef930cfc79be reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: 3cd44159-004a-40b3-9a1e-90596e648f90 kind: Event lastTimestamp: "2026-07-07T15:12:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:12:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:27Z" name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl.18c00aa303adb5e9 namespace: kserve-ci-e2e-test resourceVersion: "17185" uid: aecd0e03-cf7c-418e-9ec0-3b129c669e97 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: 3cd44159-004a-40b3-9a1e-90596e648f90 kind: Event lastTimestamp: "2026-07-07T15:12:29Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:12:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:29Z" name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl.18c00aa36b38f73e namespace: kserve-ci-e2e-test resourceVersion: "17227" uid: 5c757f05-250e-4034-b6d2-7cc42779f919 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: f793b28d-2350-42ff-94d1-769b920c38e0 kind: Event lastTimestamp: "2026-07-07T15:10:37Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c-n67nl' metadata: creationTimestamp: "2026-07-07T15:10:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:10:37Z" name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c.18c00a8958335091 namespace: kserve-ci-e2e-test resourceVersion: "16134" uid: 2d2ed8e0-e1bb-4e5c-95f9-0a99db400ecd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16101" uid: 4bcdd42f-e259-427e-9a80-59ccb74be51f kind: Event lastTimestamp: "2026-07-07T15:10:37Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-c0f7d-predictor-58bbc5c47c from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:10:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:10:37Z" name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor.18c00a89525ac6cd namespace: kserve-ci-e2e-test resourceVersion: "16104" uid: 49590400-0df6-4485-9fbf-11270b56e7eb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:10:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: 26d38942-b7b0-4722-a39a-8ff198c26ddb kind: Event lastTimestamp: "2026-07-07T15:11:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:10:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:11:07Z" name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor.18c00a8cd0eafc55 namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: 6ef3dce2-2382-4ebb-888b-13c1afd3dbe7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:10:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: 26d38942-b7b0-4722-a39a-8ff198c26ddb kind: Event lastTimestamp: "2026-07-07T15:11:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:10:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:11:07Z" name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor.18c00a8cd15357ee namespace: kserve-ci-e2e-test resourceVersion: "16484" uid: 00f6d3fd-6165-4f7a-95d1-e658c0fb71b4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:11:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16367" uid: 26d38942-b7b0-4722-a39a-8ff198c26ddb kind: Event lastTimestamp: "2026-07-07T15:12:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:11:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:12:22Z" name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor.18c00a93cf6b8fce namespace: kserve-ci-e2e-test resourceVersion: "17070" uid: 28912d02-40e1-4be6-aa20-19c510fdf33c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:11:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16367" uid: 26d38942-b7b0-4722-a39a-8ff198c26ddb kind: Event lastTimestamp: "2026-07-07T15:12:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:11:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:12:22Z" name: isvc-sklearn-graph-raw-hpa-c0f7d-predictor.18c00a93cfd7d396 namespace: kserve-ci-e2e-test resourceVersion: "17071" uid: ca1eb1c1-17d4-465f-9284-8fc65537bae5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T15:10:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-c0f7d namespace: kserve-ci-e2e-test resourceVersion: "16089" uid: 767bc8e0-c419-444d-aca2-ff00a099e6f1 kind: Event lastTimestamp: "2026-07-07T15:11:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:10:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:11:59Z" name: isvc-sklearn-graph-raw-hpa-c0f7d.18c00a895241e247 namespace: kserve-ci-e2e-test resourceVersion: "16880" uid: 043401eb-0336-48ad-9524-e362684378b2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-c0f7d namespace: kserve-ci-e2e-test resourceVersion: "16089" uid: 767bc8e0-c419-444d-aca2-ff00a099e6f1 kind: Event lastTimestamp: "2026-07-07T15:10:37Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-c0f7d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-c0f7d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:10:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:10:37Z" name: isvc-sklearn-graph-raw-hpa-c0f7d.18c00a89531264d4 namespace: kserve-ci-e2e-test resourceVersion: "16107" uid: 5db98878-67f9-438d-bc1c-60ceb9f3ea1b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:11:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-c0f7d namespace: kserve-ci-e2e-test resourceVersion: "16878" uid: 767bc8e0-c419-444d-aca2-ff00a099e6f1 kind: Event lastTimestamp: "2026-07-07T15:11:59Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-c0f7d] is Ready metadata: creationTimestamp: "2026-07-07T15:11:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:11:59Z" name: isvc-sklearn-graph-raw-hpa-c0f7d.18c00a9c748f194b namespace: kserve-ci-e2e-test resourceVersion: "16879" uid: 552f794a-2dbc-4878-adc2-cd26444a0957 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:14:13.028964Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx namespace: kserve-ci-e2e-test resourceVersion: "18110" uid: 75d74119-b14c-49a4-b638-275bf78f6536 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx to ip-10-0-143-168.ec2.internal metadata: creationTimestamp: "2026-07-07T15:14:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:14:13Z" name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx.18c00abb89fd895c namespace: kserve-ci-e2e-test resourceVersion: "18121" uid: 03c24826-5e48-4c69-af83-38152943d579 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-77985f99fc-vc8tw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:13Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx namespace: kserve-ci-e2e-test resourceVersion: "18138" uid: 75d74119-b14c-49a4-b638-275bf78f6536 kind: Event lastTimestamp: "2026-07-07T15:14:13Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:14:13Z" name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx.18c00abba48db109 namespace: kserve-ci-e2e-test resourceVersion: "18155" uid: 1d2d7368-9b7d-42f5-a4fd-eb41b0d8fc39 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx namespace: kserve-ci-e2e-test resourceVersion: "18115" uid: 75d74119-b14c-49a4-b638-275bf78f6536 kind: Event lastTimestamp: "2026-07-07T15:14:13Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:09d6f486359107a6e5d5aadd7d4fa2ea151504e2bf98f31f8912a941eb631c4b" already present on machine metadata: creationTimestamp: "2026-07-07T15:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:14:13Z" name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx.18c00abba62f9596 namespace: kserve-ci-e2e-test resourceVersion: "18158" uid: c44e6db4-108a-4320-a84f-18df20f005f2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx namespace: kserve-ci-e2e-test resourceVersion: "18115" uid: 75d74119-b14c-49a4-b638-275bf78f6536 kind: Event lastTimestamp: "2026-07-07T15:14:13Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T15:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:14:13Z" name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx.18c00abba8da9529 namespace: kserve-ci-e2e-test resourceVersion: "18160" uid: 0e771460-493d-4622-b139-4a17082bd240 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx namespace: kserve-ci-e2e-test resourceVersion: "18115" uid: 75d74119-b14c-49a4-b638-275bf78f6536 kind: Event lastTimestamp: "2026-07-07T15:14:13Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T15:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:14:13Z" name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx.18c00abba94eaceb namespace: kserve-ci-e2e-test resourceVersion: "18161" uid: 58e6159e-7e17-4ef3-971a-608a27f79d42 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx namespace: kserve-ci-e2e-test resourceVersion: "18115" uid: 75d74119-b14c-49a4-b638-275bf78f6536 kind: Event lastTimestamp: "2026-07-07T15:14:18Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:14:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:14:18Z" name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx.18c00abcca96fde4 namespace: kserve-ci-e2e-test resourceVersion: "18202" uid: 2d577a07-1d80-4bb7-818c-920ba303b3c9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx namespace: kserve-ci-e2e-test resourceVersion: "18115" uid: 75d74119-b14c-49a4-b638-275bf78f6536 kind: Event lastTimestamp: "2026-07-07T15:14:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:14:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:14:18Z" name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx.18c00abccdcfcdbb namespace: kserve-ci-e2e-test resourceVersion: "18205" uid: b08837f3-12a7-47c3-a90a-0ebbfaf48d02 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx namespace: kserve-ci-e2e-test resourceVersion: "18115" uid: 75d74119-b14c-49a4-b638-275bf78f6536 kind: Event lastTimestamp: "2026-07-07T15:14:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:14:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:14:18Z" name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx.18c00abcce42a09f namespace: kserve-ci-e2e-test resourceVersion: "18207" uid: 7aacfa4a-ef01-41d8-9797-8360c5751aed reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx namespace: kserve-ci-e2e-test resourceVersion: "18115" uid: 75d74119-b14c-49a4-b638-275bf78f6536 kind: Event lastTimestamp: "2026-07-07T15:14:18Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:14:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:14:18Z" name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx.18c00abcce4cf262 namespace: kserve-ci-e2e-test resourceVersion: "18208" uid: e0937934-5d3f-40de-96b3-c3b3146d6891 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx namespace: kserve-ci-e2e-test resourceVersion: "18115" uid: 75d74119-b14c-49a4-b638-275bf78f6536 kind: Event lastTimestamp: "2026-07-07T15:14:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:14:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:14:18Z" name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx.18c00abcd13cf2f6 namespace: kserve-ci-e2e-test resourceVersion: "18210" uid: 4d664e6e-0195-4ee0-9256-7ded13644353 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx namespace: kserve-ci-e2e-test resourceVersion: "18115" uid: 75d74119-b14c-49a4-b638-275bf78f6536 kind: Event lastTimestamp: "2026-07-07T15:14:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:14:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:14:18Z" name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx.18c00abcd1b19489 namespace: kserve-ci-e2e-test resourceVersion: "18211" uid: cc5e4d80-77ea-402a-85fe-e62f4d0eab55 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 20 eventTime: null firstTimestamp: "2026-07-07T15:14:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx namespace: kserve-ci-e2e-test resourceVersion: "18115" uid: 75d74119-b14c-49a4-b638-275bf78f6536 kind: Event lastTimestamp: "2026-07-07T15:17:11Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:14:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:11Z" name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx.18c00abe30d779fc namespace: kserve-ci-e2e-test resourceVersion: "19517" uid: 89521630-8bab-4537-b225-acbddbd9a664 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx namespace: kserve-ci-e2e-test resourceVersion: "18115" uid: 75d74119-b14c-49a4-b638-275bf78f6536 kind: Event lastTimestamp: "2026-07-07T15:17:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:03Z" name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx.18c00ae327044be6 namespace: kserve-ci-e2e-test resourceVersion: "19407" uid: 3a77029f-2312-415a-8acc-36a1501d1a58 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx namespace: kserve-ci-e2e-test resourceVersion: "18115" uid: 75d74119-b14c-49a4-b638-275bf78f6536 kind: Event lastTimestamp: "2026-07-07T15:17:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:03Z" name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx.18c00ae32704b834 namespace: kserve-ci-e2e-test resourceVersion: "19406" uid: 2bf6aaae-0e48-4fd5-9d53-65370e0a273b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:17:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx namespace: kserve-ci-e2e-test resourceVersion: "18115" uid: 75d74119-b14c-49a4-b638-275bf78f6536 kind: Event lastTimestamp: "2026-07-07T15:17:09Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:17:09Z" name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx.18c00ae371eefe39 namespace: kserve-ci-e2e-test resourceVersion: "19498" uid: e27db523-af08-400a-b701-f7447720b214 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc namespace: kserve-ci-e2e-test resourceVersion: "18088" uid: 8596bdb1-90d6-4fc2-b84c-9e58731ee579 kind: Event lastTimestamp: "2026-07-07T15:14:13Z" message: 'Created pod: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc-lkbcx' metadata: creationTimestamp: "2026-07-07T15:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:14:13Z" name: isvc-sklearn-scale-raw-11363-predictor-8dccf66dc.18c00abb89855203 namespace: kserve-ci-e2e-test resourceVersion: "18114" uid: 455c992a-7e90-4648-9918-0267a31d2695 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-11363-predictor namespace: kserve-ci-e2e-test resourceVersion: "18086" uid: 44364017-de16-4a3a-900b-8d1349119fe6 kind: Event lastTimestamp: "2026-07-07T15:14:12Z" message: Scaled up replica set isvc-sklearn-scale-raw-11363-predictor-8dccf66dc from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:14:12Z" name: isvc-sklearn-scale-raw-11363-predictor.18c00abb84549b1f namespace: kserve-ci-e2e-test resourceVersion: "18089" uid: 1d6ed77e-53d6-48ae-8cb4-a8462e0a1bab reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-11363-predictor namespace: kserve-ci-e2e-test resourceVersion: "18087" uid: 19465731-92e6-49b3-927c-0ae743615e21 kind: Event lastTimestamp: "2026-07-07T15:14:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:14:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:14:27Z" name: isvc-sklearn-scale-raw-11363-predictor.18c00abf02d8f297 namespace: kserve-ci-e2e-test resourceVersion: "18285" uid: 42090408-0948-4e9c-b0c0-85d3bf1d77ab reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-11363-predictor namespace: kserve-ci-e2e-test resourceVersion: "18087" uid: 19465731-92e6-49b3-927c-0ae743615e21 kind: Event lastTimestamp: "2026-07-07T15:14:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:14:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:14:27Z" name: isvc-sklearn-scale-raw-11363-predictor.18c00abf03486b69 namespace: kserve-ci-e2e-test resourceVersion: "18287" uid: fd47a9e1-8283-41a4-ae67-075ec8e64831 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T15:14:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-11363-predictor namespace: kserve-ci-e2e-test resourceVersion: "18286" uid: 19465731-92e6-49b3-927c-0ae743615e21 kind: Event lastTimestamp: "2026-07-07T15:16:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:14:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:16:58Z" name: isvc-sklearn-scale-raw-11363-predictor.18c00ac282922fc4 namespace: kserve-ci-e2e-test resourceVersion: "19343" uid: f79c1be0-70fa-4f44-82f4-332c1ee21e9b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T15:14:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-11363-predictor namespace: kserve-ci-e2e-test resourceVersion: "18286" uid: 19465731-92e6-49b3-927c-0ae743615e21 kind: Event lastTimestamp: "2026-07-07T15:16:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:14:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:16:58Z" name: isvc-sklearn-scale-raw-11363-predictor.18c00ac282f9141d namespace: kserve-ci-e2e-test resourceVersion: "19344" uid: dd8dc286-24c2-4de5-98d3-46ab8bebedd4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T15:14:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-11363 namespace: kserve-ci-e2e-test resourceVersion: "18077" uid: 9da0eb5b-a49c-420b-bc5f-a9e0b8cba114 kind: Event lastTimestamp: "2026-07-07T15:17:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:17:02Z" name: isvc-sklearn-scale-raw-11363.18c00abb8450443e namespace: kserve-ci-e2e-test resourceVersion: "19377" uid: 8ebf6cb6-64d6-4a96-bfc6-28e11db2f941 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-11363 namespace: kserve-ci-e2e-test resourceVersion: "18077" uid: 9da0eb5b-a49c-420b-bc5f-a9e0b8cba114 kind: Event lastTimestamp: "2026-07-07T15:14:12Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-11363": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-11363": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:14:12Z" name: isvc-sklearn-scale-raw-11363.18c00abb84eae8f0 namespace: kserve-ci-e2e-test resourceVersion: "18092" uid: 4b7c0d1c-3f92-44d7-b489-27d6cb7ef6b4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:17:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-11363 namespace: kserve-ci-e2e-test resourceVersion: "19374" uid: 9da0eb5b-a49c-420b-bc5f-a9e0b8cba114 kind: Event lastTimestamp: "2026-07-07T15:17:02Z" message: InferenceService [isvc-sklearn-scale-raw-11363] is Ready metadata: creationTimestamp: "2026-07-07T15:17:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:17:02Z" name: isvc-sklearn-scale-raw-11363.18c00ae2e30e2018 namespace: kserve-ci-e2e-test resourceVersion: "19375" uid: 2ff38127-e567-4657-a910-afa8f4dc37b8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:08:37.221240Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh namespace: kserve-ci-e2e-test resourceVersion: "15063" uid: 8ec8ada2-35b3-483d-be07-e4fe373dd681 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh to ip-10-0-143-168.ec2.internal metadata: creationTimestamp: "2026-07-07T15:08:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:08:37Z" name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh.18c00a6d5a4a43b7 namespace: kserve-ci-e2e-test resourceVersion: "15068" uid: dc72aba0-172a-4196-b26b-33a0d74afd0e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-77985f99fc-vc8tw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:37Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 8ec8ada2-35b3-483d-be07-e4fe373dd681 kind: Event lastTimestamp: "2026-07-07T15:08:37Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:08:37Z" name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh.18c00a6d740e03c0 namespace: kserve-ci-e2e-test resourceVersion: "15092" uid: 5041a1f4-4787-4e88-bc4a-61797e1592eb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh namespace: kserve-ci-e2e-test resourceVersion: "15065" uid: 8ec8ada2-35b3-483d-be07-e4fe373dd681 kind: Event lastTimestamp: "2026-07-07T15:08:37Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:09d6f486359107a6e5d5aadd7d4fa2ea151504e2bf98f31f8912a941eb631c4b" already present on machine metadata: creationTimestamp: "2026-07-07T15:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:37Z" name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh.18c00a6d75f85017 namespace: kserve-ci-e2e-test resourceVersion: "15094" uid: 5643ea9d-68a6-45c5-9aba-389ad64b9e64 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh namespace: kserve-ci-e2e-test resourceVersion: "15065" uid: 8ec8ada2-35b3-483d-be07-e4fe373dd681 kind: Event lastTimestamp: "2026-07-07T15:08:37Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T15:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:37Z" name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh.18c00a6d78ec06d4 namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: b89ae09a-0dbc-416c-89aa-d0725c060687 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh namespace: kserve-ci-e2e-test resourceVersion: "15065" uid: 8ec8ada2-35b3-483d-be07-e4fe373dd681 kind: Event lastTimestamp: "2026-07-07T15:08:37Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T15:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:37Z" name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh.18c00a6d796ff5ca namespace: kserve-ci-e2e-test resourceVersion: "15096" uid: 55bf35cd-917a-4066-94a7-3152ddd6ae88 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh namespace: kserve-ci-e2e-test resourceVersion: "15065" uid: 8ec8ada2-35b3-483d-be07-e4fe373dd681 kind: Event lastTimestamp: "2026-07-07T15:08:41Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-07T15:08:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:08:41Z" name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh.18c00a6e6ce05a8e namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: 81a253ba-0dd7-42ba-973a-d1002cbc1b2d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:09:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh namespace: kserve-ci-e2e-test resourceVersion: "15065" uid: 8ec8ada2-35b3-483d-be07-e4fe373dd681 kind: Event lastTimestamp: "2026-07-07T15:09:02Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 21.162s (21.162s including waiting). Image size: 1305547511 bytes.' metadata: creationTimestamp: "2026-07-07T15:09:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:09:02Z" name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh.18c00a735a3de2a0 namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 51d532a2-60b9-42fe-9f57-1e80cf5a7928 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:09:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh namespace: kserve-ci-e2e-test resourceVersion: "15065" uid: 8ec8ada2-35b3-483d-be07-e4fe373dd681 kind: Event lastTimestamp: "2026-07-07T15:09:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:09:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:09:03Z" name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh.18c00a735dfd70cc namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 1567c02c-aa5f-433b-9a27-2382840d2abb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:09:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh namespace: kserve-ci-e2e-test resourceVersion: "15065" uid: 8ec8ada2-35b3-483d-be07-e4fe373dd681 kind: Event lastTimestamp: "2026-07-07T15:09:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:09:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:09:03Z" name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh.18c00a735e68765e namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: 45da9e3f-a3d8-4456-94ee-f5ebf69f2b0c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:09:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh namespace: kserve-ci-e2e-test resourceVersion: "15065" uid: 8ec8ada2-35b3-483d-be07-e4fe373dd681 kind: Event lastTimestamp: "2026-07-07T15:09:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:09:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:09:03Z" name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh.18c00a735e72d47e namespace: kserve-ci-e2e-test resourceVersion: "15327" uid: d24ccab1-1614-4366-80e6-25ee69d9cc8e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:09:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh namespace: kserve-ci-e2e-test resourceVersion: "15065" uid: 8ec8ada2-35b3-483d-be07-e4fe373dd681 kind: Event lastTimestamp: "2026-07-07T15:09:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:09:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:09:03Z" name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh.18c00a73620f2c52 namespace: kserve-ci-e2e-test resourceVersion: "15329" uid: c90f74b2-2c34-44eb-8dd1-b561e8ac23a5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:09:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh namespace: kserve-ci-e2e-test resourceVersion: "15065" uid: 8ec8ada2-35b3-483d-be07-e4fe373dd681 kind: Event lastTimestamp: "2026-07-07T15:09:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:09:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:09:03Z" name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh.18c00a7362856193 namespace: kserve-ci-e2e-test resourceVersion: "15330" uid: bcdeb66c-7ee8-4029-8683-94f7f593483f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-07T15:09:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh namespace: kserve-ci-e2e-test resourceVersion: "15065" uid: 8ec8ada2-35b3-483d-be07-e4fe373dd681 kind: Event lastTimestamp: "2026-07-07T15:10:39Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:09:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:39Z" name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh.18c00a7393fd377a namespace: kserve-ci-e2e-test resourceVersion: "16239" uid: 536aa711-331c-4f23-b933-31bbf8ef7680 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh namespace: kserve-ci-e2e-test resourceVersion: "15065" uid: 8ec8ada2-35b3-483d-be07-e4fe373dd681 kind: Event lastTimestamp: "2026-07-07T15:10:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:10:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:37Z" name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh.18c00a89618a4826 namespace: kserve-ci-e2e-test resourceVersion: "16181" uid: a2aeb341-dbc3-423d-99bf-b3189019c087 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh namespace: kserve-ci-e2e-test resourceVersion: "15065" uid: 8ec8ada2-35b3-483d-be07-e4fe373dd681 kind: Event lastTimestamp: "2026-07-07T15:10:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:10:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:37Z" name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh.18c00a89618ac24d namespace: kserve-ci-e2e-test resourceVersion: "16183" uid: 2417fc13-0c02-4956-8dfa-09fa68495349 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh namespace: kserve-ci-e2e-test resourceVersion: "15065" uid: 8ec8ada2-35b3-483d-be07-e4fe373dd681 kind: Event lastTimestamp: "2026-07-07T15:10:39Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:10:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:39Z" name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh.18c00a89ee3dcfd6 namespace: kserve-ci-e2e-test resourceVersion: "16238" uid: 1e6736ce-685c-4bb6-acde-76a6a3a3afc6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989 namespace: kserve-ci-e2e-test resourceVersion: "15044" uid: 7981372d-5018-4d8c-aaf2-ae68038f00a1 kind: Event lastTimestamp: "2026-07-07T15:08:37Z" message: 'Created pod: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989-k98sh' metadata: creationTimestamp: "2026-07-07T15:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:08:37Z" name: isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989.18c00a6d59adc70d namespace: kserve-ci-e2e-test resourceVersion: "15064" uid: 1c1ce3fc-ba71-444f-b5b6-4b5035cae815 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-e3a4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: 4b648317-e719-43be-b123-1999ee9d0629 kind: Event lastTimestamp: "2026-07-07T15:08:37Z" message: Scaled up replica set isvc-xgboost-graph-raw-e3a4a-predictor-6d9989989 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:08:37Z" name: isvc-xgboost-graph-raw-e3a4a-predictor.18c00a6d5616b952 namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: b3b90592-865d-4893-bbfd-62fa16ddcabe reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:08:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-e3a4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: be953f23-670b-4ef9-ba54-cdbf7f7d8a18 kind: Event lastTimestamp: "2026-07-07T15:09:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:09:37Z" name: isvc-xgboost-graph-raw-e3a4a-predictor.18c00a70d537356a namespace: kserve-ci-e2e-test resourceVersion: "15582" uid: 55728862-8150-4264-a79e-1fe922cec83c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:08:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-e3a4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: be953f23-670b-4ef9-ba54-cdbf7f7d8a18 kind: Event lastTimestamp: "2026-07-07T15:09:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:08:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:09:37Z" name: isvc-xgboost-graph-raw-e3a4a-predictor.18c00a70d5bd58cd namespace: kserve-ci-e2e-test resourceVersion: "15583" uid: 94933905-e122-4e8f-bbf8-4397b3b55f99 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:09:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-e3a4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15239" uid: be953f23-670b-4ef9-ba54-cdbf7f7d8a18 kind: Event lastTimestamp: "2026-07-07T15:10:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:10:37Z" name: isvc-xgboost-graph-raw-e3a4a-predictor.18c00a77d36494c7 namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: 99bf34b0-1c00-418b-9f38-d427beb20897 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:09:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-e3a4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15239" uid: be953f23-670b-4ef9-ba54-cdbf7f7d8a18 kind: Event lastTimestamp: "2026-07-07T15:10:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:09:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:10:37Z" name: isvc-xgboost-graph-raw-e3a4a-predictor.18c00a77d4078b1d namespace: kserve-ci-e2e-test resourceVersion: "16091" uid: f85439a3-1d8f-4b6c-8d62-1939d33e3917 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-07T15:08:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-e3a4a namespace: kserve-ci-e2e-test resourceVersion: "15013" uid: dbe9d280-3c19-412e-ba85-d8888f80e687 kind: Event lastTimestamp: "2026-07-07T15:10:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:10:10Z" name: isvc-xgboost-graph-raw-e3a4a.18c00a6d56d7ea9d namespace: kserve-ci-e2e-test resourceVersion: "15823" uid: f3c56917-a0c5-4e2f-a28a-afb6f0574137 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:08:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-e3a4a namespace: kserve-ci-e2e-test resourceVersion: "15013" uid: dbe9d280-3c19-412e-ba85-d8888f80e687 kind: Event lastTimestamp: "2026-07-07T15:08:37Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-e3a4a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-e3a4a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:08:37Z" name: isvc-xgboost-graph-raw-e3a4a.18c00a6d57549926 namespace: kserve-ci-e2e-test resourceVersion: "15057" uid: fc98011f-aed2-4ff9-943d-0dd14e2dbc99 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-e3a4a namespace: kserve-ci-e2e-test resourceVersion: "15821" uid: dbe9d280-3c19-412e-ba85-d8888f80e687 kind: Event lastTimestamp: "2026-07-07T15:10:10Z" message: InferenceService [isvc-xgboost-graph-raw-e3a4a] is Ready metadata: creationTimestamp: "2026-07-07T15:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:10:10Z" name: isvc-xgboost-graph-raw-e3a4a.18c00a82f8295f26 namespace: kserve-ci-e2e-test resourceVersion: "15822" uid: b8668377-4ddf-492d-9119-9524075161de reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:10:37.542139Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq namespace: kserve-ci-e2e-test resourceVersion: "16154" uid: 26714cf3-2b63-4b65-90c4-6b71822b067d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq to ip-10-0-143-168.ec2.internal metadata: creationTimestamp: "2026-07-07T15:10:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:10:37Z" name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq.18c00a895df99230 namespace: kserve-ci-e2e-test resourceVersion: "16167" uid: 57e487e1-6d69-499b-9248-598e044ab671 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-77985f99fc-vc8tw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:37Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 26714cf3-2b63-4b65-90c4-6b71822b067d kind: Event lastTimestamp: "2026-07-07T15:10:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-hpa-c0f7d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T15:10:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:37Z" name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq.18c00a896af8ebad namespace: kserve-ci-e2e-test resourceVersion: "16195" uid: e54e3723-108c-4997-833c-ba4fd88a01ae reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:38Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq namespace: kserve-ci-e2e-test resourceVersion: "16177" uid: 26714cf3-2b63-4b65-90c4-6b71822b067d kind: Event lastTimestamp: "2026-07-07T15:10:38Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:10:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:10:38Z" name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq.18c00a899bde6df4 namespace: kserve-ci-e2e-test resourceVersion: "16220" uid: edc583f0-d388-4b8e-add6-48f1e6141a99 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 26714cf3-2b63-4b65-90c4-6b71822b067d kind: Event lastTimestamp: "2026-07-07T15:10:38Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:09d6f486359107a6e5d5aadd7d4fa2ea151504e2bf98f31f8912a941eb631c4b" already present on machine metadata: creationTimestamp: "2026-07-07T15:10:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:38Z" name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq.18c00a899d5da524 namespace: kserve-ci-e2e-test resourceVersion: "16222" uid: 5633a237-9072-4e3d-ab36-70d7c93776b1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 26714cf3-2b63-4b65-90c4-6b71822b067d kind: Event lastTimestamp: "2026-07-07T15:10:38Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T15:10:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:38Z" name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq.18c00a89a0e66425 namespace: kserve-ci-e2e-test resourceVersion: "16223" uid: ad1444bc-e987-4873-bd68-fb16d0f3dc3c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 26714cf3-2b63-4b65-90c4-6b71822b067d kind: Event lastTimestamp: "2026-07-07T15:10:38Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T15:10:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:38Z" name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq.18c00a89a1730477 namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: 16bf78ea-84b4-40fc-b51d-dfa75a246c4f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 26714cf3-2b63-4b65-90c4-6b71822b067d kind: Event lastTimestamp: "2026-07-07T15:10:43Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-07-07T15:10:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:43Z" name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq.18c00a8abc8bba4c namespace: kserve-ci-e2e-test resourceVersion: "16286" uid: 6058816c-c6f4-4021-b212-befa6b173f56 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 26714cf3-2b63-4b65-90c4-6b71822b067d kind: Event lastTimestamp: "2026-07-07T15:10:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:10:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:43Z" name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq.18c00a8abf6db6f1 namespace: kserve-ci-e2e-test resourceVersion: "16289" uid: d18105ca-1b57-42a8-a608-28f9e6a60895 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 26714cf3-2b63-4b65-90c4-6b71822b067d kind: Event lastTimestamp: "2026-07-07T15:10:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:10:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:43Z" name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq.18c00a8abfd0c284 namespace: kserve-ci-e2e-test resourceVersion: "16290" uid: 128b61f4-19da-49f9-8384-143c4af6840c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 26714cf3-2b63-4b65-90c4-6b71822b067d kind: Event lastTimestamp: "2026-07-07T15:10:43Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:10:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:43Z" name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq.18c00a8abfdaf41e namespace: kserve-ci-e2e-test resourceVersion: "16291" uid: 290b1bc5-7954-4ba8-9c98-bab5dc4a1908 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 26714cf3-2b63-4b65-90c4-6b71822b067d kind: Event lastTimestamp: "2026-07-07T15:10:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:10:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:43Z" name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq.18c00a8ac2d47587 namespace: kserve-ci-e2e-test resourceVersion: "16296" uid: 86f9a348-c906-4723-9bcb-96a7efbc09e4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 26714cf3-2b63-4b65-90c4-6b71822b067d kind: Event lastTimestamp: "2026-07-07T15:10:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:10:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:43Z" name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq.18c00a8ac34055c6 namespace: kserve-ci-e2e-test resourceVersion: "16297" uid: 67d67020-cb30-420c-94a7-ed5dbdf1f3eb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T15:10:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 26714cf3-2b63-4b65-90c4-6b71822b067d kind: Event lastTimestamp: "2026-07-07T15:12:31Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:10:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:31Z" name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq.18c00a8af8c1cdde namespace: kserve-ci-e2e-test resourceVersion: "17254" uid: 5ecc6d23-e642-4fdb-83d3-806e0fec7f4e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 26714cf3-2b63-4b65-90c4-6b71822b067d kind: Event lastTimestamp: "2026-07-07T15:12:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:12:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:27Z" name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq.18c00aa306162b7b namespace: kserve-ci-e2e-test resourceVersion: "17198" uid: 09a5ae60-588a-4d34-9ec8-72133d580c9f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 26714cf3-2b63-4b65-90c4-6b71822b067d kind: Event lastTimestamp: "2026-07-07T15:12:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:12:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:27Z" name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq.18c00aa30616ad7c namespace: kserve-ci-e2e-test resourceVersion: "17201" uid: 2e4e293f-2e9b-4406-8021-064e8aa2812a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 26714cf3-2b63-4b65-90c4-6b71822b067d kind: Event lastTimestamp: "2026-07-07T15:12:31Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:12:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:31Z" name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq.18c00aa3e2fc2a2f namespace: kserve-ci-e2e-test resourceVersion: "17253" uid: 1fb7a02b-58f9-41ab-9b9f-1bb0fa6cf850 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5 namespace: kserve-ci-e2e-test resourceVersion: "16140" uid: e0d7fbed-2562-4136-9397-8f4a72e2bc75 kind: Event lastTimestamp: "2026-07-07T15:10:37Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5-rctvq' metadata: creationTimestamp: "2026-07-07T15:10:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:10:37Z" name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5.18c00a895d44bd86 namespace: kserve-ci-e2e-test resourceVersion: "16156" uid: 400b1217-eef2-40ca-b570-514330c206fe reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16132" uid: d4d302b5-001f-4ec1-8793-75b823ee2d1b kind: Event lastTimestamp: "2026-07-07T15:10:37Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-c0f7d-predictor-845db67fd5 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:10:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:10:37Z" name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor.18c00a8959629372 namespace: kserve-ci-e2e-test resourceVersion: "16145" uid: 1bca5a43-daf6-4c94-8872-042cff76fa11 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:10:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: a5a2b170-60d8-4241-be36-234b0b241df1 kind: Event lastTimestamp: "2026-07-07T15:11:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:10:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:11:07Z" name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor.18c00a8cd779184f namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 12f2cbb6-abf3-4aaf-b0de-af0cf46402a1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T15:10:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: a5a2b170-60d8-4241-be36-234b0b241df1 kind: Event lastTimestamp: "2026-07-07T15:11:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:10:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:11:07Z" name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor.18c00a8cd7e82c2f namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 17645cf5-9669-4419-9943-8411ea83411f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T15:11:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16371" uid: a5a2b170-60d8-4241-be36-234b0b241df1 kind: Event lastTimestamp: "2026-07-07T15:12:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:11:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:12:07Z" name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor.18c00a93d55e96ee namespace: kserve-ci-e2e-test resourceVersion: "16939" uid: e7f89341-1190-4e73-8898-24e4d3487165 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T15:11:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16371" uid: a5a2b170-60d8-4241-be36-234b0b241df1 kind: Event lastTimestamp: "2026-07-07T15:12:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:11:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:12:07Z" name: isvc-xgboost-graph-raw-hpa-c0f7d-predictor.18c00a93d5ea5eb3 namespace: kserve-ci-e2e-test resourceVersion: "16940" uid: d0dc0a5d-8ac7-4f42-a7ad-d46b4d5ebad2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T15:10:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-c0f7d namespace: kserve-ci-e2e-test resourceVersion: "16109" uid: 535fe047-c4fb-4efd-a6f1-25d96781bb7c kind: Event lastTimestamp: "2026-07-07T15:11:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:10:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:11:51Z" name: isvc-xgboost-graph-raw-hpa-c0f7d.18c00a8959052922 namespace: kserve-ci-e2e-test resourceVersion: "16813" uid: 89a0109b-e71a-4f5b-90fe-db68eda5af94 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-c0f7d namespace: kserve-ci-e2e-test resourceVersion: "16109" uid: 535fe047-c4fb-4efd-a6f1-25d96781bb7c kind: Event lastTimestamp: "2026-07-07T15:10:37Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-hpa-c0f7d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-hpa-c0f7d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:10:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:10:37Z" name: isvc-xgboost-graph-raw-hpa-c0f7d.18c00a89599c4ce9 namespace: kserve-ci-e2e-test resourceVersion: "16148" uid: 3196155e-87e2-488f-a940-3a843ecec2ec reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:11:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-c0f7d namespace: kserve-ci-e2e-test resourceVersion: "16810" uid: 535fe047-c4fb-4efd-a6f1-25d96781bb7c kind: Event lastTimestamp: "2026-07-07T15:11:51Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-c0f7d] is Ready metadata: creationTimestamp: "2026-07-07T15:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:11:51Z" name: isvc-xgboost-graph-raw-hpa-c0f7d.18c00a9a98b0ec7b namespace: kserve-ci-e2e-test resourceVersion: "16812" uid: 416f63d4-8f17-410d-9b4c-c5fa97961538 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:12:27.669120Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-c27ac-predictor-66f488f4f4-zvdmr namespace: kserve-ci-e2e-test resourceVersion: "17160" uid: 68a8729c-ac00-4e21-aee6-b769b50f5a87 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-c27ac-predictor-66f488f4f4-zvdmr to ip-10-0-143-168.ec2.internal metadata: creationTimestamp: "2026-07-07T15:12:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:12:27Z" name: message-dumper-raw-c27ac-predictor-66f488f4f4-zvdmr.18c00aa3020de913 namespace: kserve-ci-e2e-test resourceVersion: "17168" uid: fc42da7b-e010-4567-afd5-e662cdae9fae reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-77985f99fc-vc8tw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:28Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-c27ac-predictor-66f488f4f4-zvdmr namespace: kserve-ci-e2e-test resourceVersion: "17178" uid: 68a8729c-ac00-4e21-aee6-b769b50f5a87 kind: Event lastTimestamp: "2026-07-07T15:12:28Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:12:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:12:28Z" name: message-dumper-raw-c27ac-predictor-66f488f4f4-zvdmr.18c00aa31bf432a2 namespace: kserve-ci-e2e-test resourceVersion: "17209" uid: 8ec8a764-91c0-4e72-b7b9-79cf7d389403 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-c27ac-predictor-66f488f4f4-zvdmr namespace: kserve-ci-e2e-test resourceVersion: "17166" uid: 68a8729c-ac00-4e21-aee6-b769b50f5a87 kind: Event lastTimestamp: "2026-07-07T15:12:28Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-07T15:12:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:28Z" name: message-dumper-raw-c27ac-predictor-66f488f4f4-zvdmr.18c00aa31d6aae9e namespace: kserve-ci-e2e-test resourceVersion: "17211" uid: 2a991160-48a5-4dd7-8b13-712eaee803da reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-c27ac-predictor-66f488f4f4-zvdmr namespace: kserve-ci-e2e-test resourceVersion: "17166" uid: 68a8729c-ac00-4e21-aee6-b769b50f5a87 kind: Event lastTimestamp: "2026-07-07T15:12:29Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.443s (1.443s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-07T15:12:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:29Z" name: message-dumper-raw-c27ac-predictor-66f488f4f4-zvdmr.18c00aa37379dc98 namespace: kserve-ci-e2e-test resourceVersion: "17230" uid: 897e2d19-803c-4a3f-910a-60989401960f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-c27ac-predictor-66f488f4f4-zvdmr namespace: kserve-ci-e2e-test resourceVersion: "17166" uid: 68a8729c-ac00-4e21-aee6-b769b50f5a87 kind: Event lastTimestamp: "2026-07-07T15:12:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:12:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:29Z" name: message-dumper-raw-c27ac-predictor-66f488f4f4-zvdmr.18c00aa376905d53 namespace: kserve-ci-e2e-test resourceVersion: "17231" uid: c4537df2-1b1a-487a-a4c5-4893c5d54995 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-c27ac-predictor-66f488f4f4-zvdmr namespace: kserve-ci-e2e-test resourceVersion: "17166" uid: 68a8729c-ac00-4e21-aee6-b769b50f5a87 kind: Event lastTimestamp: "2026-07-07T15:12:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:12:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:29Z" name: message-dumper-raw-c27ac-predictor-66f488f4f4-zvdmr.18c00aa3770d73f3 namespace: kserve-ci-e2e-test resourceVersion: "17232" uid: a32c097e-12c8-4b58-85b5-c885a9b4ddbb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-c27ac-predictor-66f488f4f4-zvdmr namespace: kserve-ci-e2e-test resourceVersion: "17166" uid: 68a8729c-ac00-4e21-aee6-b769b50f5a87 kind: Event lastTimestamp: "2026-07-07T15:12:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:12:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:29Z" name: message-dumper-raw-c27ac-predictor-66f488f4f4-zvdmr.18c00aa37718c9ae namespace: kserve-ci-e2e-test resourceVersion: "17233" uid: 4084a646-3e8c-4aa3-84ce-6a568f1caabd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-c27ac-predictor-66f488f4f4-zvdmr namespace: kserve-ci-e2e-test resourceVersion: "17166" uid: 68a8729c-ac00-4e21-aee6-b769b50f5a87 kind: Event lastTimestamp: "2026-07-07T15:12:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:12:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:29Z" name: message-dumper-raw-c27ac-predictor-66f488f4f4-zvdmr.18c00aa37a9316a2 namespace: kserve-ci-e2e-test resourceVersion: "17234" uid: f05d3893-2a4e-4efb-a7cf-166e80ee0324 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-c27ac-predictor-66f488f4f4-zvdmr namespace: kserve-ci-e2e-test resourceVersion: "17166" uid: 68a8729c-ac00-4e21-aee6-b769b50f5a87 kind: Event lastTimestamp: "2026-07-07T15:12:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:12:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:29Z" name: message-dumper-raw-c27ac-predictor-66f488f4f4-zvdmr.18c00aa37afb89bc namespace: kserve-ci-e2e-test resourceVersion: "17235" uid: cdcc2eab-4307-4874-9329-fcf86b6f9d11 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-c27ac-predictor-66f488f4f4-zvdmr namespace: kserve-ci-e2e-test resourceVersion: "17166" uid: 68a8729c-ac00-4e21-aee6-b769b50f5a87 kind: Event lastTimestamp: "2026-07-07T15:14:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:14:13Z" name: message-dumper-raw-c27ac-predictor-66f488f4f4-zvdmr.18c00abb8ca001e2 namespace: kserve-ci-e2e-test resourceVersion: "18140" uid: ce739c64-6891-4c2e-9251-304cbd284753 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:14:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-c27ac-predictor-66f488f4f4-zvdmr namespace: kserve-ci-e2e-test resourceVersion: "17166" uid: 68a8729c-ac00-4e21-aee6-b769b50f5a87 kind: Event lastTimestamp: "2026-07-07T15:14:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:14:13Z" name: message-dumper-raw-c27ac-predictor-66f488f4f4-zvdmr.18c00abb8ca129d2 namespace: kserve-ci-e2e-test resourceVersion: "18141" uid: 601433c7-4581-4635-b1f7-97a659d9fe39 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-c27ac-predictor-66f488f4f4 namespace: kserve-ci-e2e-test resourceVersion: "17146" uid: e75862ee-7001-409b-9e9b-7acfb4e5ef79 kind: Event lastTimestamp: "2026-07-07T15:12:27Z" message: 'Created pod: message-dumper-raw-c27ac-predictor-66f488f4f4-zvdmr' metadata: creationTimestamp: "2026-07-07T15:12:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:12:27Z" name: message-dumper-raw-c27ac-predictor-66f488f4f4.18c00aa301a4c720 namespace: kserve-ci-e2e-test resourceVersion: "17165" uid: 4d1177d1-7706-461e-b5ba-1bc40f6223ce reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-c27ac-predictor namespace: kserve-ci-e2e-test resourceVersion: "17144" uid: da83c34b-3580-44bd-9659-83d9500c3e2e kind: Event lastTimestamp: "2026-07-07T15:12:27Z" message: Scaled up replica set message-dumper-raw-c27ac-predictor-66f488f4f4 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:12:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:12:27Z" name: message-dumper-raw-c27ac-predictor.18c00aa2feaafca4 namespace: kserve-ci-e2e-test resourceVersion: "17148" uid: 473d6aa9-2347-4f74-b7a1-3a577b4ea56a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-c27ac-predictor namespace: kserve-ci-e2e-test resourceVersion: "17145" uid: 87a9f9ba-0880-4c77-82b4-e187ab55d39b kind: Event lastTimestamp: "2026-07-07T15:12:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:12:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:12:42Z" name: message-dumper-raw-c27ac-predictor.18c00aa67d52e011 namespace: kserve-ci-e2e-test resourceVersion: "17392" uid: 301264cc-b8bb-47ff-b2b9-09b6df52c055 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-c27ac-predictor namespace: kserve-ci-e2e-test resourceVersion: "17145" uid: 87a9f9ba-0880-4c77-82b4-e187ab55d39b kind: Event lastTimestamp: "2026-07-07T15:12:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:12:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:12:42Z" name: message-dumper-raw-c27ac-predictor.18c00aa67dc01695 namespace: kserve-ci-e2e-test resourceVersion: "17394" uid: b44e5775-2c6e-4a69-8b25-61ecd3f63e4d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-c27ac-predictor namespace: kserve-ci-e2e-test resourceVersion: "17393" uid: 87a9f9ba-0880-4c77-82b4-e187ab55d39b kind: Event lastTimestamp: "2026-07-07T15:12:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:12:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:12:57Z" name: message-dumper-raw-c27ac-predictor.18c00aa9fc677c62 namespace: kserve-ci-e2e-test resourceVersion: "17533" uid: 10b1a0ba-75fd-4e93-ad4b-986bdb37828d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-c27ac-predictor namespace: kserve-ci-e2e-test resourceVersion: "17393" uid: 87a9f9ba-0880-4c77-82b4-e187ab55d39b kind: Event lastTimestamp: "2026-07-07T15:12:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:12:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:12:57Z" name: message-dumper-raw-c27ac-predictor.18c00aa9fcd74af1 namespace: kserve-ci-e2e-test resourceVersion: "17535" uid: 009cc5b5-f3fd-4cbe-9147-e874a29c8836 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-07T15:12:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-c27ac namespace: kserve-ci-e2e-test resourceVersion: "17136" uid: cf21ea72-a679-46e6-ac7e-0936e78527b8 kind: Event lastTimestamp: "2026-07-07T15:12:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:12:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:12:37Z" name: message-dumper-raw-c27ac.18c00aa2fe78853b namespace: kserve-ci-e2e-test resourceVersion: "17305" uid: 72d9382e-2617-459b-a86b-35de99c20675 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-c27ac namespace: kserve-ci-e2e-test resourceVersion: "17136" uid: cf21ea72-a679-46e6-ac7e-0936e78527b8 kind: Event lastTimestamp: "2026-07-07T15:12:27Z" message: 'Failed to update status for InferenceService "message-dumper-raw-c27ac": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-c27ac": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:12:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:12:27Z" name: message-dumper-raw-c27ac.18c00aa2fec99681 namespace: kserve-ci-e2e-test resourceVersion: "17150" uid: b2a8898e-5923-4d03-ab13-6cbcfd195fcd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-c27ac namespace: kserve-ci-e2e-test resourceVersion: "17303" uid: cf21ea72-a679-46e6-ac7e-0936e78527b8 kind: Event lastTimestamp: "2026-07-07T15:12:36Z" message: InferenceService [message-dumper-raw-c27ac] is Ready metadata: creationTimestamp: "2026-07-07T15:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:12:36Z" name: message-dumper-raw-c27ac.18c00aa52e1651c5 namespace: kserve-ci-e2e-test resourceVersion: "17304" uid: 0a2d36aa-5535-4951-ac96-da3ba87cac28 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:10:27.142175Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-e3a4a-9c8cdb665-bslgn namespace: kserve-ci-e2e-test resourceVersion: "15972" uid: 1733cdfe-0472-4d0a-a6af-9838c69d758e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-e3a4a-9c8cdb665-bslgn to ip-10-0-138-241.ec2.internal metadata: creationTimestamp: "2026-07-07T15:10:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:10:27Z" name: model-chainer-raw-e3a4a-9c8cdb665-bslgn.18c00a86f216ab37 namespace: kserve-ci-e2e-test resourceVersion: "15977" uid: 2c703c3d-13af-4c7b-b89b-2ccf43749536 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-77985f99fc-vc8tw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:27Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-e3a4a-9c8cdb665-bslgn namespace: kserve-ci-e2e-test resourceVersion: "15980" uid: 1733cdfe-0472-4d0a-a6af-9838c69d758e kind: Event lastTimestamp: "2026-07-07T15:10:27Z" message: Add eth0 [10.133.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:10:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:10:27Z" name: model-chainer-raw-e3a4a-9c8cdb665-bslgn.18c00a870b92529e namespace: kserve-ci-e2e-test resourceVersion: "15990" uid: a83ce6d5-8626-481f-bbcb-4cb38a16a3b2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-e3a4a} kind: Pod name: model-chainer-raw-e3a4a-9c8cdb665-bslgn namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 1733cdfe-0472-4d0a-a6af-9838c69d758e kind: Event lastTimestamp: "2026-07-07T15:10:27Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:b3cb5f47c4ea9d876e5490bc83b15040508a0aac484b7e9d11de2faf2dbcdfe4" metadata: creationTimestamp: "2026-07-07T15:10:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:27Z" name: model-chainer-raw-e3a4a-9c8cdb665-bslgn.18c00a870d0ca0d4 namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: 62053797-0d75-4966-b8b2-f2f2faab1e13 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-241.ec2.internal source: component: kubelet host: ip-10-0-138-241.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-e3a4a} kind: Pod name: model-chainer-raw-e3a4a-9c8cdb665-bslgn namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 1733cdfe-0472-4d0a-a6af-9838c69d758e kind: Event lastTimestamp: "2026-07-07T15:10:29Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:b3cb5f47c4ea9d876e5490bc83b15040508a0aac484b7e9d11de2faf2dbcdfe4" in 2.293s (2.293s including waiting). Image size: 218072672 bytes.' metadata: creationTimestamp: "2026-07-07T15:10:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:29Z" name: model-chainer-raw-e3a4a-9c8cdb665-bslgn.18c00a8795c1d1a0 namespace: kserve-ci-e2e-test resourceVersion: "16005" uid: edf9d59f-2f8f-4fec-b38e-b9a507f55809 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-241.ec2.internal source: component: kubelet host: ip-10-0-138-241.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-e3a4a} kind: Pod name: model-chainer-raw-e3a4a-9c8cdb665-bslgn namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 1733cdfe-0472-4d0a-a6af-9838c69d758e kind: Event lastTimestamp: "2026-07-07T15:10:29Z" message: 'Created container: model-chainer-raw-e3a4a' metadata: creationTimestamp: "2026-07-07T15:10:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:29Z" name: model-chainer-raw-e3a4a-9c8cdb665-bslgn.18c00a879961b8ac namespace: kserve-ci-e2e-test resourceVersion: "16007" uid: 491d8723-17e3-416f-8019-6b27ddf7f27c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-241.ec2.internal source: component: kubelet host: ip-10-0-138-241.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-e3a4a} kind: Pod name: model-chainer-raw-e3a4a-9c8cdb665-bslgn namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 1733cdfe-0472-4d0a-a6af-9838c69d758e kind: Event lastTimestamp: "2026-07-07T15:10:29Z" message: Started container model-chainer-raw-e3a4a metadata: creationTimestamp: "2026-07-07T15:10:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:29Z" name: model-chainer-raw-e3a4a-9c8cdb665-bslgn.18c00a8799d0201a namespace: kserve-ci-e2e-test resourceVersion: "16008" uid: 72ec5561-e5c7-41fb-a0e7-2bc77db1297b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-241.ec2.internal source: component: kubelet host: ip-10-0-138-241.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-e3a4a} kind: Pod name: model-chainer-raw-e3a4a-9c8cdb665-bslgn namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 1733cdfe-0472-4d0a-a6af-9838c69d758e kind: Event lastTimestamp: "2026-07-07T15:10:37Z" message: Stopping container model-chainer-raw-e3a4a metadata: creationTimestamp: "2026-07-07T15:10:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:10:37Z" name: model-chainer-raw-e3a4a-9c8cdb665-bslgn.18c00a894a418d33 namespace: kserve-ci-e2e-test resourceVersion: "16082" uid: 90bad417-c933-410b-b984-041fa3d32436 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-241.ec2.internal source: component: kubelet host: ip-10-0-138-241.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-07T15:10:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-e3a4a} kind: Pod name: model-chainer-raw-e3a4a-9c8cdb665-bslgn namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 1733cdfe-0472-4d0a-a6af-9838c69d758e kind: Event lastTimestamp: "2026-07-07T15:11:07Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T15:10:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:11:07Z" name: model-chainer-raw-e3a4a-9c8cdb665-bslgn.18c00a894a95a5e0 namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: 5615b707-d6d1-441e-8295-4d2c16da12be reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-241.ec2.internal source: component: kubelet host: ip-10-0-138-241.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-e3a4a-9c8cdb665 namespace: kserve-ci-e2e-test resourceVersion: "15969" uid: 4fd4110c-900d-4417-aeb9-76f6886e54f9 kind: Event lastTimestamp: "2026-07-07T15:10:27Z" message: 'Created pod: model-chainer-raw-e3a4a-9c8cdb665-bslgn' metadata: creationTimestamp: "2026-07-07T15:10:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:10:27Z" name: model-chainer-raw-e3a4a-9c8cdb665.18c00a86f1ae38b5 namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: 679304db-0338-478c-9c06-aa0c3f0bc604 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-e3a4a namespace: kserve-ci-e2e-test resourceVersion: "15967" uid: d8e87c63-813c-4172-9487-96c9c9b69171 kind: Event lastTimestamp: "2026-07-07T15:10:27Z" message: Scaled up replica set model-chainer-raw-e3a4a-9c8cdb665 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:10:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:10:27Z" name: model-chainer-raw-e3a4a.18c00a86f09ae15c namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: 34750fa6-b3e4-4e8b-8da1-4e54aeebfabc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:10:27Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-e3a4a namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: febbd797-3322-402b-a062-e81fb57c16a7 kind: Event lastTimestamp: "2026-07-07T15:10:27Z" message: InferenceGraph [model-chainer-raw-e3a4a] is Ready metadata: creationTimestamp: "2026-07-07T15:10:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:10:27Z" name: model-chainer-raw-e3a4a.18c00a86f63e6861 namespace: kserve-ci-e2e-test resourceVersion: "15985" uid: d0197f2e-a4f4-4072-afc9-a98f67a224b9 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:12:17.435037Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-c0f7d-74cb8888c-bklx7 namespace: kserve-ci-e2e-test resourceVersion: "17016" uid: 805dd105-b477-4560-b2dd-c86e0f091cde kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-c0f7d-74cb8888c-bklx7 to ip-10-0-138-241.ec2.internal metadata: creationTimestamp: "2026-07-07T15:12:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:12:17Z" name: model-chainer-raw-hpa-c0f7d-74cb8888c-bklx7.18c00aa0a00e2129 namespace: kserve-ci-e2e-test resourceVersion: "17021" uid: bf1cc3c1-7ed0-44b2-a776-3b49cdc64a7b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-77985f99fc-vc8tw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:17Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-c0f7d-74cb8888c-bklx7 namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 805dd105-b477-4560-b2dd-c86e0f091cde kind: Event lastTimestamp: "2026-07-07T15:12:17Z" message: Add eth0 [10.133.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:12:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:12:17Z" name: model-chainer-raw-hpa-c0f7d-74cb8888c-bklx7.18c00aa0b99b0186 namespace: kserve-ci-e2e-test resourceVersion: "17035" uid: 73733cfb-d2fd-4858-8b97-d247d7d9cf6d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-c0f7d} kind: Pod name: model-chainer-raw-hpa-c0f7d-74cb8888c-bklx7 namespace: kserve-ci-e2e-test resourceVersion: "17017" uid: 805dd105-b477-4560-b2dd-c86e0f091cde kind: Event lastTimestamp: "2026-07-07T15:12:17Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b3cb5f47c4ea9d876e5490bc83b15040508a0aac484b7e9d11de2faf2dbcdfe4" already present on machine metadata: creationTimestamp: "2026-07-07T15:12:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:17Z" name: model-chainer-raw-hpa-c0f7d-74cb8888c-bklx7.18c00aa0bae56c97 namespace: kserve-ci-e2e-test resourceVersion: "17037" uid: 7e8d3708-2e4a-4820-a258-0752f47f17e3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-241.ec2.internal source: component: kubelet host: ip-10-0-138-241.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-c0f7d} kind: Pod name: model-chainer-raw-hpa-c0f7d-74cb8888c-bklx7 namespace: kserve-ci-e2e-test resourceVersion: "17017" uid: 805dd105-b477-4560-b2dd-c86e0f091cde kind: Event lastTimestamp: "2026-07-07T15:12:17Z" message: 'Created container: model-chainer-raw-hpa-c0f7d' metadata: creationTimestamp: "2026-07-07T15:12:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:17Z" name: model-chainer-raw-hpa-c0f7d-74cb8888c-bklx7.18c00aa0bdfef4eb namespace: kserve-ci-e2e-test resourceVersion: "17039" uid: 43f7ae55-5a7b-46fd-a1f6-0162d4363474 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-241.ec2.internal source: component: kubelet host: ip-10-0-138-241.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-c0f7d} kind: Pod name: model-chainer-raw-hpa-c0f7d-74cb8888c-bklx7 namespace: kserve-ci-e2e-test resourceVersion: "17017" uid: 805dd105-b477-4560-b2dd-c86e0f091cde kind: Event lastTimestamp: "2026-07-07T15:12:17Z" message: Started container model-chainer-raw-hpa-c0f7d metadata: creationTimestamp: "2026-07-07T15:12:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:17Z" name: model-chainer-raw-hpa-c0f7d-74cb8888c-bklx7.18c00aa0be716a0c namespace: kserve-ci-e2e-test resourceVersion: "17040" uid: bda9b6b4-a4f1-4d5a-99e1-3d6b69866538 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-241.ec2.internal source: component: kubelet host: ip-10-0-138-241.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-c0f7d} kind: Pod name: model-chainer-raw-hpa-c0f7d-74cb8888c-bklx7 namespace: kserve-ci-e2e-test resourceVersion: "17017" uid: 805dd105-b477-4560-b2dd-c86e0f091cde kind: Event lastTimestamp: "2026-07-07T15:12:27Z" message: Stopping container model-chainer-raw-hpa-c0f7d metadata: creationTimestamp: "2026-07-07T15:12:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:27Z" name: model-chainer-raw-hpa-c0f7d-74cb8888c-bklx7.18c00aa2f64f2c1d namespace: kserve-ci-e2e-test resourceVersion: "17129" uid: 7ee042a6-5238-46d3-a331-978081fb1a2d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-241.ec2.internal source: component: kubelet host: ip-10-0-138-241.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T15:12:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-c0f7d} kind: Pod name: model-chainer-raw-hpa-c0f7d-74cb8888c-bklx7 namespace: kserve-ci-e2e-test resourceVersion: "17017" uid: 805dd105-b477-4560-b2dd-c86e0f091cde kind: Event lastTimestamp: "2026-07-07T15:12:54Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T15:12:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:12:54Z" name: model-chainer-raw-hpa-c0f7d-74cb8888c-bklx7.18c00aa387fc08d5 namespace: kserve-ci-e2e-test resourceVersion: "17507" uid: a109b33c-d0e6-4dac-9ef8-1af548443134 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-241.ec2.internal source: component: kubelet host: ip-10-0-138-241.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-c0f7d-74cb8888c namespace: kserve-ci-e2e-test resourceVersion: "17013" uid: 4f962d42-d52c-4436-883b-e6bc061a6c49 kind: Event lastTimestamp: "2026-07-07T15:12:17Z" message: 'Created pod: model-chainer-raw-hpa-c0f7d-74cb8888c-bklx7' metadata: creationTimestamp: "2026-07-07T15:12:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:12:17Z" name: model-chainer-raw-hpa-c0f7d-74cb8888c.18c00aa09fb47e48 namespace: kserve-ci-e2e-test resourceVersion: "17018" uid: b42cc25b-aec5-46e8-83b7-05fc6d79e255 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-c0f7d namespace: kserve-ci-e2e-test resourceVersion: "17010" uid: 12becd50-3de6-4299-ac9a-37033063b692 kind: Event lastTimestamp: "2026-07-07T15:12:17Z" message: Scaled up replica set model-chainer-raw-hpa-c0f7d-74cb8888c from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:12:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:12:17Z" name: model-chainer-raw-hpa-c0f7d.18c00aa09ea74a11 namespace: kserve-ci-e2e-test resourceVersion: "17014" uid: 5e1c812d-85b2-45b2-930d-51dd6f69a70d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:12:17Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-c0f7d namespace: kserve-ci-e2e-test resourceVersion: "17030" uid: e541b330-67a7-4e5b-b98e-4ab216abec72 kind: Event lastTimestamp: "2026-07-07T15:12:17Z" message: InferenceGraph [model-chainer-raw-hpa-c0f7d] is Ready metadata: creationTimestamp: "2026-07-07T15:12:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:12:17Z" name: model-chainer-raw-hpa-c0f7d.18c00aa0a4b48842 namespace: kserve-ci-e2e-test resourceVersion: "17032" uid: 15020721-2c7f-41bf-82d8-92c93628722f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:19:16.938562Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m namespace: kserve-ci-e2e-test resourceVersion: "20637" uid: f3baa47e-215e-4f27-81d4-7874722cfdec kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m to ip-10-0-143-168.ec2.internal metadata: creationTimestamp: "2026-07-07T15:19:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:19:16Z" name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m.18c00b024c69fc3e namespace: kserve-ci-e2e-test resourceVersion: "20643" uid: 2427d775-1755-484d-8d76-33cec992d414 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-77985f99fc-vc8tw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:17Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m namespace: kserve-ci-e2e-test resourceVersion: "20647" uid: f3baa47e-215e-4f27-81d4-7874722cfdec kind: Event lastTimestamp: "2026-07-07T15:19:17Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:19:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:19:17Z" name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m.18c00b0266979439 namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 6fa1b669-40b6-4b6b-a27d-aabcb84cae85 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m namespace: kserve-ci-e2e-test resourceVersion: "20641" uid: f3baa47e-215e-4f27-81d4-7874722cfdec kind: Event lastTimestamp: "2026-07-07T15:19:17Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:09d6f486359107a6e5d5aadd7d4fa2ea151504e2bf98f31f8912a941eb631c4b" already present on machine metadata: creationTimestamp: "2026-07-07T15:19:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:19:17Z" name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m.18c00b0268290ee1 namespace: kserve-ci-e2e-test resourceVersion: "20656" uid: 84d13d0c-047b-4cf9-a588-20031e511e9a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m namespace: kserve-ci-e2e-test resourceVersion: "20641" uid: f3baa47e-215e-4f27-81d4-7874722cfdec kind: Event lastTimestamp: "2026-07-07T15:19:17Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T15:19:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:19:17Z" name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m.18c00b026aed0ce7 namespace: kserve-ci-e2e-test resourceVersion: "20657" uid: 081b1f87-2178-4f32-9e6c-3a4a3af888bf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m namespace: kserve-ci-e2e-test resourceVersion: "20641" uid: f3baa47e-215e-4f27-81d4-7874722cfdec kind: Event lastTimestamp: "2026-07-07T15:19:17Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T15:19:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:19:17Z" name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m.18c00b026b623a8b namespace: kserve-ci-e2e-test resourceVersion: "20658" uid: 65307c25-5322-4b7f-a935-229385d6a299 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m namespace: kserve-ci-e2e-test resourceVersion: "20641" uid: f3baa47e-215e-4f27-81d4-7874722cfdec kind: Event lastTimestamp: "2026-07-07T15:19:21Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:19:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:19:21Z" name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m.18c00b036bce3091 namespace: kserve-ci-e2e-test resourceVersion: "20696" uid: c00cfddc-5cae-4f84-bd34-844713e39d17 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m namespace: kserve-ci-e2e-test resourceVersion: "20641" uid: f3baa47e-215e-4f27-81d4-7874722cfdec kind: Event lastTimestamp: "2026-07-07T15:19:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:19:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:19:21Z" name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m.18c00b036e903200 namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: 144fd414-c26f-4e5c-9e52-56ff036112fe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m namespace: kserve-ci-e2e-test resourceVersion: "20641" uid: f3baa47e-215e-4f27-81d4-7874722cfdec kind: Event lastTimestamp: "2026-07-07T15:19:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:19:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:19:21Z" name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m.18c00b036f029039 namespace: kserve-ci-e2e-test resourceVersion: "20699" uid: ae0e70c4-c03e-4d6e-9622-b064fd2af480 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m namespace: kserve-ci-e2e-test resourceVersion: "20641" uid: f3baa47e-215e-4f27-81d4-7874722cfdec kind: Event lastTimestamp: "2026-07-07T15:19:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:19:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:19:21Z" name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m.18c00b036f0e8749 namespace: kserve-ci-e2e-test resourceVersion: "20700" uid: 1fb39912-8c39-465f-83ec-724bda32600f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m namespace: kserve-ci-e2e-test resourceVersion: "20641" uid: f3baa47e-215e-4f27-81d4-7874722cfdec kind: Event lastTimestamp: "2026-07-07T15:19:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:19:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:19:21Z" name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m.18c00b03726c4447 namespace: kserve-ci-e2e-test resourceVersion: "20703" uid: b9cde525-d377-400a-91f5-c3fb504e5f33 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m namespace: kserve-ci-e2e-test resourceVersion: "20641" uid: f3baa47e-215e-4f27-81d4-7874722cfdec kind: Event lastTimestamp: "2026-07-07T15:19:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:19:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:19:21Z" name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m.18c00b0372e240a7 namespace: kserve-ci-e2e-test resourceVersion: "20704" uid: fab7a9fb-800d-4905-ac74-f481dd54ebd2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T15:19:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m namespace: kserve-ci-e2e-test resourceVersion: "20641" uid: f3baa47e-215e-4f27-81d4-7874722cfdec kind: Event lastTimestamp: "2026-07-07T15:20:49Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:19:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:20:49Z" name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m.18c00b03e3be8570 namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: ed11bd64-defc-47a0-ae2d-05833e77c13c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:20:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m namespace: kserve-ci-e2e-test resourceVersion: "20641" uid: f3baa47e-215e-4f27-81d4-7874722cfdec kind: Event lastTimestamp: "2026-07-07T15:20:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:20:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:20:47Z" name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m.18c00b17579a63dd namespace: kserve-ci-e2e-test resourceVersion: "21335" uid: 7d602004-72be-4d22-a4f6-6782871d3d4d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:20:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m namespace: kserve-ci-e2e-test resourceVersion: "20641" uid: f3baa47e-215e-4f27-81d4-7874722cfdec kind: Event lastTimestamp: "2026-07-07T15:20:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:20:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:20:47Z" name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m.18c00b17579af8ac namespace: kserve-ci-e2e-test resourceVersion: "21336" uid: 70664f24-50d6-490c-ac33-c5b61bff5602 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:20:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m namespace: kserve-ci-e2e-test resourceVersion: "20641" uid: f3baa47e-215e-4f27-81d4-7874722cfdec kind: Event lastTimestamp: "2026-07-07T15:20:49Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp 10.134.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:20:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:20:49Z" name: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m.18c00b17ea003980 namespace: kserve-ci-e2e-test resourceVersion: "21351" uid: fc68b82b-3ad4-4cb5-babd-d0a6b638a26e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-2e704-predictor-bb8b4dfb5 namespace: kserve-ci-e2e-test resourceVersion: "20628" uid: 1e080ea1-5c58-4f5e-9299-a54516448d84 kind: Event lastTimestamp: "2026-07-07T15:19:16Z" message: 'Created pod: raw-sklearn-2e704-predictor-bb8b4dfb5-7b52m' metadata: creationTimestamp: "2026-07-07T15:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:19:16Z" name: raw-sklearn-2e704-predictor-bb8b4dfb5.18c00b024bfe9336 namespace: kserve-ci-e2e-test resourceVersion: "20640" uid: a6d81824-ff5e-49a1-b6c1-e8a90b817a08 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-2e704-predictor namespace: kserve-ci-e2e-test resourceVersion: "20626" uid: a507c39f-a1d7-47a9-8acd-3824001c5462 kind: Event lastTimestamp: "2026-07-07T15:19:16Z" message: Scaled up replica set raw-sklearn-2e704-predictor-bb8b4dfb5 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:19:16Z" name: raw-sklearn-2e704-predictor.18c00b0248f835a1 namespace: kserve-ci-e2e-test resourceVersion: "20630" uid: 47b82e05-f760-4dee-b3db-85b115e31663 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:19:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-2e704-predictor namespace: kserve-ci-e2e-test resourceVersion: "20627" uid: dda9b822-ac8b-4892-9ec5-e0af8a1d3ada kind: Event lastTimestamp: "2026-07-07T15:20:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:19:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:20:01Z" name: raw-sklearn-2e704-predictor.18c00b05c831310a namespace: kserve-ci-e2e-test resourceVersion: "20986" uid: 79c226e0-7c4e-406d-af5b-b7da4caf0cb2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:19:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-2e704-predictor namespace: kserve-ci-e2e-test resourceVersion: "20627" uid: dda9b822-ac8b-4892-9ec5-e0af8a1d3ada kind: Event lastTimestamp: "2026-07-07T15:20:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:19:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:20:01Z" name: raw-sklearn-2e704-predictor.18c00b05c8f9c97b namespace: kserve-ci-e2e-test resourceVersion: "20987" uid: 79aeba77-6be6-466f-a31f-abde1251a03a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:20:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-2e704-predictor namespace: kserve-ci-e2e-test resourceVersion: "20783" uid: dda9b822-ac8b-4892-9ec5-e0af8a1d3ada kind: Event lastTimestamp: "2026-07-07T15:20:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:20:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:20:47Z" name: raw-sklearn-2e704-predictor.18c00b10471c1a79 namespace: kserve-ci-e2e-test resourceVersion: "21314" uid: 45f1f43e-d60e-48d2-b2fe-161838549a32 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T15:20:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-2e704-predictor namespace: kserve-ci-e2e-test resourceVersion: "20783" uid: dda9b822-ac8b-4892-9ec5-e0af8a1d3ada kind: Event lastTimestamp: "2026-07-07T15:20:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:20:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:20:47Z" name: raw-sklearn-2e704-predictor.18c00b1047918f18 namespace: kserve-ci-e2e-test resourceVersion: "21315" uid: 92f68d21-a28d-422a-a6cb-ecb5808d6683 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T15:19:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-2e704 namespace: kserve-ci-e2e-test resourceVersion: "20619" uid: 7cf65514-0e30-438d-b6ca-b46072244151 kind: Event lastTimestamp: "2026-07-07T15:20:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:20:39Z" name: raw-sklearn-2e704.18c00b0248f6a00d namespace: kserve-ci-e2e-test resourceVersion: "21261" uid: b0a749a2-7bb9-4f50-abe8-054c1e1b7717 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:19:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-2e704 namespace: kserve-ci-e2e-test resourceVersion: "20619" uid: 7cf65514-0e30-438d-b6ca-b46072244151 kind: Event lastTimestamp: "2026-07-07T15:19:16Z" message: 'Failed to update status for InferenceService "raw-sklearn-2e704": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-2e704": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:19:16Z" name: raw-sklearn-2e704.18c00b02498afc60 namespace: kserve-ci-e2e-test resourceVersion: "20633" uid: 6b87142b-0a30-4fcb-bced-6a790e7b604a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:20:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-2e704 namespace: kserve-ci-e2e-test resourceVersion: "21259" uid: 7cf65514-0e30-438d-b6ca-b46072244151 kind: Event lastTimestamp: "2026-07-07T15:20:39Z" message: InferenceService [raw-sklearn-2e704] is Ready metadata: creationTimestamp: "2026-07-07T15:20:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:20:39Z" name: raw-sklearn-2e704.18c00b159e7de376 namespace: kserve-ci-e2e-test resourceVersion: "21260" uid: c30586b2-5d2b-43a4-9709-3e30d808a867 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T15:20:52.335441Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922 namespace: kserve-ci-e2e-test resourceVersion: "21394" uid: 69a1ebc8-7d5e-4f82-9591-e20585bff1cf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-00f07-predictor-5977594885-pc922 to ip-10-0-143-168.ec2.internal metadata: creationTimestamp: "2026-07-07T15:20:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T15:20:52Z" name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922.18c00b188282d468 namespace: kserve-ci-e2e-test resourceVersion: "21399" uid: eadc6cae-9274-4849-91ed-7c4c615753d3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-77985f99fc-vc8tw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:20:52Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922 namespace: kserve-ci-e2e-test resourceVersion: "21403" uid: 69a1ebc8-7d5e-4f82-9591-e20585bff1cf kind: Event lastTimestamp: "2026-07-07T15:20:52Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T15:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T15:20:52Z" name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922.18c00b189c53aab7 namespace: kserve-ci-e2e-test resourceVersion: "21409" uid: e187f81f-27a7-4dca-b8c3-cf8cf32493b0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922 namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: 69a1ebc8-7d5e-4f82-9591-e20585bff1cf kind: Event lastTimestamp: "2026-07-07T15:20:52Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:09d6f486359107a6e5d5aadd7d4fa2ea151504e2bf98f31f8912a941eb631c4b" already present on machine metadata: creationTimestamp: "2026-07-07T15:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:20:52Z" name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922.18c00b189ddf7eb2 namespace: kserve-ci-e2e-test resourceVersion: "21411" uid: 11e5d074-904d-4b6f-b654-c7cdb04dfc02 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922 namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: 69a1ebc8-7d5e-4f82-9591-e20585bff1cf kind: Event lastTimestamp: "2026-07-07T15:20:52Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T15:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:20:52Z" name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922.18c00b18a0a410b4 namespace: kserve-ci-e2e-test resourceVersion: "21412" uid: ed0e453c-3d3e-43ba-8f4c-f9531b67ad28 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922 namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: 69a1ebc8-7d5e-4f82-9591-e20585bff1cf kind: Event lastTimestamp: "2026-07-07T15:20:52Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T15:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:20:52Z" name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922.18c00b18a11a8624 namespace: kserve-ci-e2e-test resourceVersion: "21413" uid: 45625299-333e-41d1-bd9b-50e91bd46d2f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922 namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: 69a1ebc8-7d5e-4f82-9591-e20585bff1cf kind: Event lastTimestamp: "2026-07-07T15:20:57Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-07T15:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:20:57Z" name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922.18c00b19a272c7d4 namespace: kserve-ci-e2e-test resourceVersion: "21454" uid: 1858e61f-8c97-4546-a1ce-8776300a5438 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922 namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: 69a1ebc8-7d5e-4f82-9591-e20585bff1cf kind: Event lastTimestamp: "2026-07-07T15:20:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T15:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:20:57Z" name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922.18c00b19a565d421 namespace: kserve-ci-e2e-test resourceVersion: "21457" uid: 1dfa0574-eb92-47bc-b895-9f67619191f5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922 namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: 69a1ebc8-7d5e-4f82-9591-e20585bff1cf kind: Event lastTimestamp: "2026-07-07T15:20:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T15:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:20:57Z" name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922.18c00b19a5d98106 namespace: kserve-ci-e2e-test resourceVersion: "21458" uid: fb666e4f-fa62-48ac-9f20-a87d285101ed reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922 namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: 69a1ebc8-7d5e-4f82-9591-e20585bff1cf kind: Event lastTimestamp: "2026-07-07T15:20:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T15:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:20:57Z" name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922.18c00b19a5e40fba namespace: kserve-ci-e2e-test resourceVersion: "21460" uid: 3834f42f-ba65-4409-a4d2-b519ea42defd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922 namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: 69a1ebc8-7d5e-4f82-9591-e20585bff1cf kind: Event lastTimestamp: "2026-07-07T15:20:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T15:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:20:57Z" name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922.18c00b19a8c39f84 namespace: kserve-ci-e2e-test resourceVersion: "21462" uid: cc1ece2c-6818-4f8f-8074-bcf33cb28081 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922 namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: 69a1ebc8-7d5e-4f82-9591-e20585bff1cf kind: Event lastTimestamp: "2026-07-07T15:20:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:20:57Z" name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922.18c00b19a9382c18 namespace: kserve-ci-e2e-test resourceVersion: "21463" uid: 7a0d06d7-f063-46b5-b277-ad85f44446fc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T15:20:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922 namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: 69a1ebc8-7d5e-4f82-9591-e20585bff1cf kind: Event lastTimestamp: "2026-07-07T15:22:25Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:20:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:22:25Z" name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922.18c00b1a1aa72ae6 namespace: kserve-ci-e2e-test resourceVersion: "22110" uid: ae6cd5ee-413f-4c69-99b5-898a9d52d8b0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:22:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922 namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: 69a1ebc8-7d5e-4f82-9591-e20585bff1cf kind: Event lastTimestamp: "2026-07-07T15:22:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T15:22:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:22:22Z" name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922.18c00b2d7d44715c namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: c25b2201-6069-4f30-9abd-2c49c28f7f14 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:22:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922 namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: 69a1ebc8-7d5e-4f82-9591-e20585bff1cf kind: Event lastTimestamp: "2026-07-07T15:22:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T15:22:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:22:22Z" name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922.18c00b2d7d478e44 namespace: kserve-ci-e2e-test resourceVersion: "22089" uid: 42395a2c-a658-4203-831c-24facbbad37b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:22:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922 namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: 69a1ebc8-7d5e-4f82-9591-e20585bff1cf kind: Event lastTimestamp: "2026-07-07T15:22:25Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T15:22:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T15:22:25Z" name: raw-sklearn-runtime-00f07-predictor-5977594885-pc922.18c00b2e20e0a64d namespace: kserve-ci-e2e-test resourceVersion: "22109" uid: adbdfb7c-60ce-4799-9c9f-a54015994ee9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-168.ec2.internal source: component: kubelet host: ip-10-0-143-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:20:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-00f07-predictor-5977594885 namespace: kserve-ci-e2e-test resourceVersion: "21380" uid: 8c448266-9eb3-4130-9885-63dbee2f37f2 kind: Event lastTimestamp: "2026-07-07T15:20:52Z" message: 'Created pod: raw-sklearn-runtime-00f07-predictor-5977594885-pc922' metadata: creationTimestamp: "2026-07-07T15:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:20:52Z" name: raw-sklearn-runtime-00f07-predictor-5977594885.18c00b1881cdcbc8 namespace: kserve-ci-e2e-test resourceVersion: "21395" uid: bf568881-4fc4-4e5f-8bce-c2783544ecd1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:20:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-00f07-predictor namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: 7bacd2c0-1380-4665-be52-c116d72e19bd kind: Event lastTimestamp: "2026-07-07T15:20:52Z" message: Scaled up replica set raw-sklearn-runtime-00f07-predictor-5977594885 from 0 to 1 metadata: creationTimestamp: "2026-07-07T15:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:20:52Z" name: raw-sklearn-runtime-00f07-predictor.18c00b187ada2ba4 namespace: kserve-ci-e2e-test resourceVersion: "21382" uid: bc083199-1570-4611-8681-698e0b71256c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:21:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-00f07-predictor namespace: kserve-ci-e2e-test resourceVersion: "21379" uid: afa0fccc-2e3f-4f06-8be6-901feaf82b44 kind: Event lastTimestamp: "2026-07-07T15:21:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:21:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:21:07Z" name: raw-sklearn-runtime-00f07-predictor.18c00b1bfa0aa910 namespace: kserve-ci-e2e-test resourceVersion: "21537" uid: 7cdbd3a5-92b6-4db2-8fe3-ee62329c26cd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:21:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-00f07-predictor namespace: kserve-ci-e2e-test resourceVersion: "21379" uid: afa0fccc-2e3f-4f06-8be6-901feaf82b44 kind: Event lastTimestamp: "2026-07-07T15:21:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T15:21:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:21:07Z" name: raw-sklearn-runtime-00f07-predictor.18c00b1bfa8b163b namespace: kserve-ci-e2e-test resourceVersion: "21539" uid: 85388d7c-757e-49dd-9781-2e2db45f576c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:21:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-00f07-predictor namespace: kserve-ci-e2e-test resourceVersion: "21538" uid: afa0fccc-2e3f-4f06-8be6-901feaf82b44 kind: Event lastTimestamp: "2026-07-07T15:22:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:21:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:22:22Z" name: raw-sklearn-runtime-00f07-predictor.18c00b1f79e51678 namespace: kserve-ci-e2e-test resourceVersion: "22072" uid: 8e52826d-3e1a-4b71-9ffc-78c940349fc5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T15:21:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-00f07-predictor namespace: kserve-ci-e2e-test resourceVersion: "21538" uid: afa0fccc-2e3f-4f06-8be6-901feaf82b44 kind: Event lastTimestamp: "2026-07-07T15:22:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T15:21:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T15:22:22Z" name: raw-sklearn-runtime-00f07-predictor.18c00b1f7a992f56 namespace: kserve-ci-e2e-test resourceVersion: "22073" uid: f7821c89-d7d7-4678-8fd8-2d4258d33779 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T15:20:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-00f07 namespace: kserve-ci-e2e-test resourceVersion: "21370" uid: 88ec47e1-028b-4333-8846-f6c2666c9b5b kind: Event lastTimestamp: "2026-07-07T15:22:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T15:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:22:15Z" name: raw-sklearn-runtime-00f07.18c00b187adca970 namespace: kserve-ci-e2e-test resourceVersion: "22022" uid: c3d46022-0615-4cb0-8049-a5175463bef7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:20:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-00f07 namespace: kserve-ci-e2e-test resourceVersion: "21370" uid: 88ec47e1-028b-4333-8846-f6c2666c9b5b kind: Event lastTimestamp: "2026-07-07T15:20:52Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-00f07": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-00f07": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T15:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:20:52Z" name: raw-sklearn-runtime-00f07.18c00b187b9e7f73 namespace: kserve-ci-e2e-test resourceVersion: "21385" uid: 76f393fc-e21f-4e00-b0b0-ff6eeffcefb1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T15:22:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-00f07 namespace: kserve-ci-e2e-test resourceVersion: "22020" uid: 88ec47e1-028b-4333-8846-f6c2666c9b5b kind: Event lastTimestamp: "2026-07-07T15:22:15Z" message: InferenceService [raw-sklearn-runtime-00f07] is Ready metadata: creationTimestamp: "2026-07-07T15:22:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T15:22:15Z" name: raw-sklearn-runtime-00f07.18c00b2bd280334e namespace: kserve-ci-e2e-test resourceVersion: "22021" uid: a64e57a9-600a-4048-a9ef-f8e2e4a683e6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "22346"