--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:37:03.883255Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-0ecf21-predictor-777dcd9689-sphbp namespace: kserve-ci-e2e-test resourceVersion: "20571" uid: 933efd14-d42d-467d-a7d1-3ae478991c3a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-0ecf21-predictor-777dcd9689-sphbp to ip-10-0-132-82.ec2.internal metadata: creationTimestamp: "2026-07-02T14:37:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:37:03Z" name: isvc-init-fail-0ecf21-predictor-777dcd9689-sphbp.18be7fcdb0024e1d namespace: kserve-ci-e2e-test resourceVersion: "20577" uid: c9be05b2-13f4-490f-9c1b-8e1cc556e0b0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75fd45cf98-dpdjc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:04Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-0ecf21-predictor-777dcd9689-sphbp namespace: kserve-ci-e2e-test resourceVersion: "20581" uid: 933efd14-d42d-467d-a7d1-3ae478991c3a kind: Event lastTimestamp: "2026-07-02T14:37:04Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:37:04Z" name: isvc-init-fail-0ecf21-predictor-777dcd9689-sphbp.18be7fcdc947f7a2 namespace: kserve-ci-e2e-test resourceVersion: "20589" uid: 9bd911c0-1429-46e2-ac78-5270743a3f7c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:37:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-0ecf21-predictor-777dcd9689-sphbp namespace: kserve-ci-e2e-test resourceVersion: "20575" uid: 933efd14-d42d-467d-a7d1-3ae478991c3a kind: Event lastTimestamp: "2026-07-02T14:37:07Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8e2b89e7009febc315fad3ff8d06e5aec68b35b7cc22df79daf7c1334f909f5a" already present on machine metadata: creationTimestamp: "2026-07-02T14:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:07Z" name: isvc-init-fail-0ecf21-predictor-777dcd9689-sphbp.18be7fcdcabc0718 namespace: kserve-ci-e2e-test resourceVersion: "20622" uid: 18beb49e-227e-467b-889d-05c4d0dff8c0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:37:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-0ecf21-predictor-777dcd9689-sphbp namespace: kserve-ci-e2e-test resourceVersion: "20575" uid: 933efd14-d42d-467d-a7d1-3ae478991c3a kind: Event lastTimestamp: "2026-07-02T14:37:07Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:07Z" name: isvc-init-fail-0ecf21-predictor-777dcd9689-sphbp.18be7fcdcdb88028 namespace: kserve-ci-e2e-test resourceVersion: "20625" uid: 34d34651-848e-47e8-ba31-764cc6c55f12 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:37:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-0ecf21-predictor-777dcd9689-sphbp namespace: kserve-ci-e2e-test resourceVersion: "20575" uid: 933efd14-d42d-467d-a7d1-3ae478991c3a kind: Event lastTimestamp: "2026-07-02T14:37:07Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:07Z" name: isvc-init-fail-0ecf21-predictor-777dcd9689-sphbp.18be7fcdce2be0e2 namespace: kserve-ci-e2e-test resourceVersion: "20626" uid: 08d8b757-7148-49ac-a879-ea251caa82bf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:37:08Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-0ecf21-predictor-777dcd9689-sphbp namespace: kserve-ci-e2e-test resourceVersion: "20575" uid: 933efd14-d42d-467d-a7d1-3ae478991c3a kind: Event lastTimestamp: "2026-07-02T14:37:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-init-fail-0ecf21-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:37:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:10Z" name: isvc-init-fail-0ecf21-predictor-777dcd9689-sphbp.18be7fcedaa0d55e namespace: kserve-ci-e2e-test resourceVersion: "20672" uid: c5cb9810-0cdc-49fd-94b7-401453d08068 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-0ecf21-predictor-777dcd9689-sphbp namespace: kserve-ci-e2e-test resourceVersion: "20575" uid: 933efd14-d42d-467d-a7d1-3ae478991c3a kind: Event lastTimestamp: "2026-07-02T14:37:09Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-07-02T14:37:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:09Z" name: isvc-init-fail-0ecf21-predictor-777dcd9689-sphbp.18be7fcf1119cf5d namespace: kserve-ci-e2e-test resourceVersion: "20664" uid: 78c7dcdb-41e4-456c-b462-9951a288714d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-0ecf21-predictor-777dcd9689 namespace: kserve-ci-e2e-test resourceVersion: "20562" uid: a507faa9-130c-48ea-8d8e-4a012abf09a9 kind: Event lastTimestamp: "2026-07-02T14:37:03Z" message: 'Created pod: isvc-init-fail-0ecf21-predictor-777dcd9689-sphbp' metadata: creationTimestamp: "2026-07-02T14:37:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:37:03Z" name: isvc-init-fail-0ecf21-predictor-777dcd9689.18be7fcdaf62ee9a namespace: kserve-ci-e2e-test resourceVersion: "20574" uid: ca55f298-0214-494a-99eb-eb6fa04f8603 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-0ecf21-predictor namespace: kserve-ci-e2e-test resourceVersion: "20560" uid: 7aaba7a8-4bf3-4ae6-98ef-7f7574c8236e kind: Event lastTimestamp: "2026-07-02T14:37:03Z" message: Scaled up replica set isvc-init-fail-0ecf21-predictor-777dcd9689 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:37:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:37:03Z" name: isvc-init-fail-0ecf21-predictor.18be7fcdac65110b namespace: kserve-ci-e2e-test resourceVersion: "20563" uid: 7fde940c-8109-4173-baa5-fd17bb34e7c0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T14:37:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-0ecf21 namespace: kserve-ci-e2e-test resourceVersion: "20550" uid: d89751b5-90a1-4d3c-ac84-66245216f05e kind: Event lastTimestamp: "2026-07-02T14:37:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:37:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:37:07Z" name: isvc-init-fail-0ecf21.18be7fcdac661735 namespace: kserve-ci-e2e-test resourceVersion: "20629" uid: 3f8c6d28-d6c6-4083-8777-a2ff187882cd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-0ecf21 namespace: kserve-ci-e2e-test resourceVersion: "20550" uid: d89751b5-90a1-4d3c-ac84-66245216f05e kind: Event lastTimestamp: "2026-07-02T14:37:03Z" message: 'Failed to update status for InferenceService "isvc-init-fail-0ecf21": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-0ecf21": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:37:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:37:03Z" name: isvc-init-fail-0ecf21.18be7fcdacbd7917 namespace: kserve-ci-e2e-test resourceVersion: "20567" uid: 60c7688e-084b-4c4a-8f49-4b035b284653 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-0ecf21 namespace: kserve-ci-e2e-test resourceVersion: "20550" uid: d89751b5-90a1-4d3c-ac84-66245216f05e kind: Event lastTimestamp: "2026-07-02T14:37:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-0ecf21": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:37:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:37:03Z" name: isvc-init-fail-0ecf21.18be7fcdacbdd8b2 namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: 855a8b21-1f43-4f3c-a257-a37ceae10480 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:30:47.827613Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh namespace: kserve-ci-e2e-test resourceVersion: "17510" uid: 6fee74b3-e784-45c0-82bb-c2cfe9c68c68 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh to ip-10-0-132-82.ec2.internal metadata: creationTimestamp: "2026-07-02T14:30:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:30:47Z" name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh.18be7f762158ae71 namespace: kserve-ci-e2e-test resourceVersion: "17514" uid: 591f219e-0b6a-4d79-86c8-b1f94802bc2c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75fd45cf98-dpdjc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:48Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh namespace: kserve-ci-e2e-test resourceVersion: "17518" uid: 6fee74b3-e784-45c0-82bb-c2cfe9c68c68 kind: Event lastTimestamp: "2026-07-02T14:30:48Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:30:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:30:48Z" name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh.18be7f763ac8a8cc namespace: kserve-ci-e2e-test resourceVersion: "17527" uid: 91d9e5ec-9112-40c6-b856-b4c64d98d7ae reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh namespace: kserve-ci-e2e-test resourceVersion: "17512" uid: 6fee74b3-e784-45c0-82bb-c2cfe9c68c68 kind: Event lastTimestamp: "2026-07-02T14:30:48Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8e2b89e7009febc315fad3ff8d06e5aec68b35b7cc22df79daf7c1334f909f5a" already present on machine metadata: creationTimestamp: "2026-07-02T14:30:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:48Z" name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh.18be7f763c562257 namespace: kserve-ci-e2e-test resourceVersion: "17529" uid: bde7688c-fad7-4c34-8254-7016dcd0e740 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh namespace: kserve-ci-e2e-test resourceVersion: "17512" uid: 6fee74b3-e784-45c0-82bb-c2cfe9c68c68 kind: Event lastTimestamp: "2026-07-02T14:30:48Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:30:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:48Z" name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh.18be7f763eee1428 namespace: kserve-ci-e2e-test resourceVersion: "17530" uid: 315a8435-dd15-495b-a93a-02704d4dc6ae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh namespace: kserve-ci-e2e-test resourceVersion: "17512" uid: 6fee74b3-e784-45c0-82bb-c2cfe9c68c68 kind: Event lastTimestamp: "2026-07-02T14:30:48Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:30:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:48Z" name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh.18be7f763f70a46d namespace: kserve-ci-e2e-test resourceVersion: "17531" uid: 12a1e312-fe6e-462f-a6af-50f355886be2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh namespace: kserve-ci-e2e-test resourceVersion: "17512" uid: 6fee74b3-e784-45c0-82bb-c2cfe9c68c68 kind: Event lastTimestamp: "2026-07-02T14:30:52Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:30:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:52Z" name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh.18be7f7733cd1694 namespace: kserve-ci-e2e-test resourceVersion: "17568" uid: 3334ca1c-535d-4456-b1de-01ba2b87160b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh namespace: kserve-ci-e2e-test resourceVersion: "17512" uid: 6fee74b3-e784-45c0-82bb-c2cfe9c68c68 kind: Event lastTimestamp: "2026-07-02T14:30:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:30:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:52Z" name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh.18be7f77372a3d54 namespace: kserve-ci-e2e-test resourceVersion: "17570" uid: 492831ad-5776-4031-afbe-078873a07a22 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh namespace: kserve-ci-e2e-test resourceVersion: "17512" uid: 6fee74b3-e784-45c0-82bb-c2cfe9c68c68 kind: Event lastTimestamp: "2026-07-02T14:30:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:30:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:52Z" name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh.18be7f77379f9f45 namespace: kserve-ci-e2e-test resourceVersion: "17571" uid: 5cc407f6-cbf9-4d38-8dab-2acb43c4c016 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh namespace: kserve-ci-e2e-test resourceVersion: "17512" uid: 6fee74b3-e784-45c0-82bb-c2cfe9c68c68 kind: Event lastTimestamp: "2026-07-02T14:30:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:30:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:52Z" name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh.18be7f7737aa5dc6 namespace: kserve-ci-e2e-test resourceVersion: "17572" uid: e2cef21f-8284-42d7-9eff-6e7133a58500 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh namespace: kserve-ci-e2e-test resourceVersion: "17512" uid: 6fee74b3-e784-45c0-82bb-c2cfe9c68c68 kind: Event lastTimestamp: "2026-07-02T14:30:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:30:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:52Z" name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh.18be7f773a9256d7 namespace: kserve-ci-e2e-test resourceVersion: "17576" uid: b28d10a5-678c-4375-8080-53f5181b79bc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh namespace: kserve-ci-e2e-test resourceVersion: "17512" uid: 6fee74b3-e784-45c0-82bb-c2cfe9c68c68 kind: Event lastTimestamp: "2026-07-02T14:30:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:30:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:52Z" name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh.18be7f773b00d946 namespace: kserve-ci-e2e-test resourceVersion: "17579" uid: ac861aac-6d25-4107-8a7e-38cb28bf7115 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh namespace: kserve-ci-e2e-test resourceVersion: "17512" uid: 6fee74b3-e784-45c0-82bb-c2cfe9c68c68 kind: Event lastTimestamp: "2026-07-02T14:30:52Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:e129573a9a25e59fc4e926ae207b9073f8d71d58bb61a455387cef8a5c19e72b" already present on machine metadata: creationTimestamp: "2026-07-02T14:30:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:52Z" name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh.18be7f773b0bc648 namespace: kserve-ci-e2e-test resourceVersion: "17580" uid: 989ce68b-72eb-4523-af33-7aa2d8c8c721 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh namespace: kserve-ci-e2e-test resourceVersion: "17512" uid: 6fee74b3-e784-45c0-82bb-c2cfe9c68c68 kind: Event lastTimestamp: "2026-07-02T14:30:52Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-02T14:30:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:52Z" name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh.18be7f773e3cfab1 namespace: kserve-ci-e2e-test resourceVersion: "17581" uid: e4b59bb1-d345-48ce-a051-cf6f5759fc22 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh namespace: kserve-ci-e2e-test resourceVersion: "17512" uid: 6fee74b3-e784-45c0-82bb-c2cfe9c68c68 kind: Event lastTimestamp: "2026-07-02T14:30:52Z" message: Started container agent metadata: creationTimestamp: "2026-07-02T14:30:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:52Z" name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh.18be7f773eac68c5 namespace: kserve-ci-e2e-test resourceVersion: "17582" uid: 0ad38975-eea0-4c08-939a-78a9244ccfef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T14:30:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh namespace: kserve-ci-e2e-test resourceVersion: "17512" uid: 6fee74b3-e784-45c0-82bb-c2cfe9c68c68 kind: Event lastTimestamp: "2026-07-02T14:32:40Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:30:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:32:40Z" name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh.18be7f77ab9e5a7d namespace: kserve-ci-e2e-test resourceVersion: "18486" uid: c6b41ae8-a70b-4512-8384-973e39390edd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T14:30:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh namespace: kserve-ci-e2e-test resourceVersion: "17512" uid: 6fee74b3-e784-45c0-82bb-c2cfe9c68c68 kind: Event lastTimestamp: "2026-07-02T14:32:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:30:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:32:30Z" name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh.18be7f77aba82887 namespace: kserve-ci-e2e-test resourceVersion: "18407" uid: 200eafe6-9b48-4f93-94b2-558e862d7321 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh namespace: kserve-ci-e2e-test resourceVersion: "17512" uid: 6fee74b3-e784-45c0-82bb-c2cfe9c68c68 kind: Event lastTimestamp: "2026-07-02T14:32:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:32:23Z" name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh.18be7f8c4e020540 namespace: kserve-ci-e2e-test resourceVersion: "18279" uid: 248b50e2-abca-4e31-b673-811bd5b6a539 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh namespace: kserve-ci-e2e-test resourceVersion: "17512" uid: 6fee74b3-e784-45c0-82bb-c2cfe9c68c68 kind: Event lastTimestamp: "2026-07-02T14:32:23Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-02T14:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:32:23Z" name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh.18be7f8c4e02abcf namespace: kserve-ci-e2e-test resourceVersion: "18283" uid: caa140c3-7b0b-479b-bb7c-6b15923ce197 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh namespace: kserve-ci-e2e-test resourceVersion: "17512" uid: 6fee74b3-e784-45c0-82bb-c2cfe9c68c68 kind: Event lastTimestamp: "2026-07-02T14:32:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:32:23Z" name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh.18be7f8c4e02e7bb namespace: kserve-ci-e2e-test resourceVersion: "18285" uid: b666216f-f2b7-490c-a74d-c789abddf9b0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:32:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh namespace: kserve-ci-e2e-test resourceVersion: "17512" uid: 6fee74b3-e784-45c0-82bb-c2cfe9c68c68 kind: Event lastTimestamp: "2026-07-02T14:32:40Z" message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp 10.134.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:32:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:32:40Z" name: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh.18be7f8cdbd35058 namespace: kserve-ci-e2e-test resourceVersion: "18485" uid: 06dc15b0-6ea9-4276-a18c-cc19fed2bd31 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-f89aa-predictor-55d69dbdfc namespace: kserve-ci-e2e-test resourceVersion: "17499" uid: fe6a9d30-e1fe-438b-811e-340b5f6fbe42 kind: Event lastTimestamp: "2026-07-02T14:30:47Z" message: 'Created pod: isvc-logger-raw-f89aa-predictor-55d69dbdfc-s8qqh' metadata: creationTimestamp: "2026-07-02T14:30:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:30:47Z" name: isvc-logger-raw-f89aa-predictor-55d69dbdfc.18be7f7620eb2f17 namespace: kserve-ci-e2e-test resourceVersion: "17511" uid: 6eae8448-a073-4c93-9fed-b6ee97525da7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-f89aa-predictor namespace: kserve-ci-e2e-test resourceVersion: "17497" uid: c69a2a38-a8da-4e45-9501-f605e38d52da kind: Event lastTimestamp: "2026-07-02T14:30:47Z" message: Scaled up replica set isvc-logger-raw-f89aa-predictor-55d69dbdfc from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:30:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:30:47Z" name: isvc-logger-raw-f89aa-predictor.18be7f761d596519 namespace: kserve-ci-e2e-test resourceVersion: "17500" uid: 0658cbba-c797-4312-8efd-fc910cf8346f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:31:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-f89aa-predictor namespace: kserve-ci-e2e-test resourceVersion: "17498" uid: a03dff7a-1de0-4cfe-bc34-66df1d6dc7fb kind: Event lastTimestamp: "2026-07-02T14:31:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:31:17Z" name: isvc-logger-raw-f89aa-predictor.18be7f799bd2bbd3 namespace: kserve-ci-e2e-test resourceVersion: "17778" uid: 3095077e-0afd-4d73-be2e-6d2ded79e626 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:31:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-f89aa-predictor namespace: kserve-ci-e2e-test resourceVersion: "17498" uid: a03dff7a-1de0-4cfe-bc34-66df1d6dc7fb kind: Event lastTimestamp: "2026-07-02T14:31:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:31:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:31:17Z" name: isvc-logger-raw-f89aa-predictor.18be7f799c351793 namespace: kserve-ci-e2e-test resourceVersion: "17779" uid: 85010a1c-a9af-4cde-8d3c-ae938d4e1c20 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:31:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-f89aa-predictor namespace: kserve-ci-e2e-test resourceVersion: "17664" uid: a03dff7a-1de0-4cfe-bc34-66df1d6dc7fb kind: Event lastTimestamp: "2026-07-02T14:32:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:32:17Z" name: isvc-logger-raw-f89aa-predictor.18be7f809a0829b7 namespace: kserve-ci-e2e-test resourceVersion: "18209" uid: 8f407893-6fc7-402f-a166-9e39860140ad reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:31:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-f89aa-predictor namespace: kserve-ci-e2e-test resourceVersion: "17664" uid: a03dff7a-1de0-4cfe-bc34-66df1d6dc7fb kind: Event lastTimestamp: "2026-07-02T14:32:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:32:17Z" name: isvc-logger-raw-f89aa-predictor.18be7f809a7acc9c namespace: kserve-ci-e2e-test resourceVersion: "18210" uid: 66b23971-2cd3-4a44-84b4-699e13d128f2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T14:30:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-f89aa namespace: kserve-ci-e2e-test resourceVersion: "17490" uid: 9b4f3755-fa3b-448d-ba9c-b2828b266e2b kind: Event lastTimestamp: "2026-07-02T14:32:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:30:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:32:10Z" name: isvc-logger-raw-f89aa.18be7f761d59d9c9 namespace: kserve-ci-e2e-test resourceVersion: "18159" uid: 01741ed6-b5e3-459b-968d-3c7c77fd034b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-f89aa namespace: kserve-ci-e2e-test resourceVersion: "17490" uid: 9b4f3755-fa3b-448d-ba9c-b2828b266e2b kind: Event lastTimestamp: "2026-07-02T14:30:47Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-f89aa": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-f89aa": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:30:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:30:47Z" name: isvc-logger-raw-f89aa.18be7f761ddcf3fa namespace: kserve-ci-e2e-test resourceVersion: "17503" uid: 63405a3d-cc3d-4e0b-8a4f-6134bb42ea03 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:32:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-f89aa namespace: kserve-ci-e2e-test resourceVersion: "18157" uid: 9b4f3755-fa3b-448d-ba9c-b2828b266e2b kind: Event lastTimestamp: "2026-07-02T14:32:10Z" message: InferenceService [isvc-logger-raw-f89aa] is Ready metadata: creationTimestamp: "2026-07-02T14:32:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:32:10Z" name: isvc-logger-raw-f89aa.18be7f89649c2033 namespace: kserve-ci-e2e-test resourceVersion: "18158" uid: f0a5094c-db82-42ea-8a35-c9afefbc95ba reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:35:08.438577Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f namespace: kserve-ci-e2e-test resourceVersion: "19568" uid: b4b8e2c5-d5ef-4f95-8130-fe918dade8b6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-571456-predictor-bdc8f5d66-rdw9f to ip-10-0-132-82.ec2.internal metadata: creationTimestamp: "2026-07-02T14:35:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:35:08Z" name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f.18be7fb2cef86936 namespace: kserve-ci-e2e-test resourceVersion: "19573" uid: 5c6aff98-a56e-406c-93c2-88d8e43f13aa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75fd45cf98-dpdjc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:35:08Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f namespace: kserve-ci-e2e-test resourceVersion: "19578" uid: b4b8e2c5-d5ef-4f95-8130-fe918dade8b6 kind: Event lastTimestamp: "2026-07-02T14:35:08Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:35:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:35:08Z" name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f.18be7fb2e863ff00 namespace: kserve-ci-e2e-test resourceVersion: "19589" uid: 9a086b2b-d0b1-41a7-b85c-c2f825ec5e64 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:35:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f namespace: kserve-ci-e2e-test resourceVersion: "19570" uid: b4b8e2c5-d5ef-4f95-8130-fe918dade8b6 kind: Event lastTimestamp: "2026-07-02T14:35:08Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8e2b89e7009febc315fad3ff8d06e5aec68b35b7cc22df79daf7c1334f909f5a" already present on machine metadata: creationTimestamp: "2026-07-02T14:35:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:35:08Z" name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f.18be7fb2e9d6ee2e namespace: kserve-ci-e2e-test resourceVersion: "19591" uid: 2034619c-7db5-41f1-a38b-870c1a347cd6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:35:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f namespace: kserve-ci-e2e-test resourceVersion: "19570" uid: b4b8e2c5-d5ef-4f95-8130-fe918dade8b6 kind: Event lastTimestamp: "2026-07-02T14:35:08Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:35:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:35:08Z" name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f.18be7fb2ec9947ce namespace: kserve-ci-e2e-test resourceVersion: "19592" uid: 6a777c4a-4e34-4b56-991a-1531cf05d21b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:35:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f namespace: kserve-ci-e2e-test resourceVersion: "19570" uid: b4b8e2c5-d5ef-4f95-8130-fe918dade8b6 kind: Event lastTimestamp: "2026-07-02T14:35:08Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:35:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:35:08Z" name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f.18be7fb2ed08713c namespace: kserve-ci-e2e-test resourceVersion: "19593" uid: 4056f50f-1fa7-4e0b-95fe-00c8549fb962 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:35:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f namespace: kserve-ci-e2e-test resourceVersion: "19570" uid: b4b8e2c5-d5ef-4f95-8130-fe918dade8b6 kind: Event lastTimestamp: "2026-07-02T14:35:13Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:35:13Z" name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f.18be7fb3f56ca832 namespace: kserve-ci-e2e-test resourceVersion: "19633" uid: f54e1fc7-c27b-4725-9c91-5d1037cbb16f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:35:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f namespace: kserve-ci-e2e-test resourceVersion: "19570" uid: b4b8e2c5-d5ef-4f95-8130-fe918dade8b6 kind: Event lastTimestamp: "2026-07-02T14:35:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:35:13Z" name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f.18be7fb3f8b711cd namespace: kserve-ci-e2e-test resourceVersion: "19639" uid: dfd788aa-d380-49fc-840f-ba8de1e8a987 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:35:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f namespace: kserve-ci-e2e-test resourceVersion: "19570" uid: b4b8e2c5-d5ef-4f95-8130-fe918dade8b6 kind: Event lastTimestamp: "2026-07-02T14:35:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:35:13Z" name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f.18be7fb3f92b6454 namespace: kserve-ci-e2e-test resourceVersion: "19642" uid: 1d7d4656-618a-46d4-b9f9-2aaf9ff52d82 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:35:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f namespace: kserve-ci-e2e-test resourceVersion: "19570" uid: b4b8e2c5-d5ef-4f95-8130-fe918dade8b6 kind: Event lastTimestamp: "2026-07-02T14:35:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:35:13Z" name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f.18be7fb3f934e966 namespace: kserve-ci-e2e-test resourceVersion: "19643" uid: b2d55401-6228-4b87-8587-7f740f2ba0a6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:35:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f namespace: kserve-ci-e2e-test resourceVersion: "19570" uid: b4b8e2c5-d5ef-4f95-8130-fe918dade8b6 kind: Event lastTimestamp: "2026-07-02T14:35:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:35:13Z" name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f.18be7fb3fc171b9e namespace: kserve-ci-e2e-test resourceVersion: "19646" uid: d5a47806-3131-49a1-8b67-49e59bdeb499 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:35:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f namespace: kserve-ci-e2e-test resourceVersion: "19570" uid: b4b8e2c5-d5ef-4f95-8130-fe918dade8b6 kind: Event lastTimestamp: "2026-07-02T14:35:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:35:13Z" name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f.18be7fb3fc873bac namespace: kserve-ci-e2e-test resourceVersion: "19647" uid: 9b52d126-ddd1-4cc3-a008-0d94869d6551 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T14:35:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f namespace: kserve-ci-e2e-test resourceVersion: "19570" uid: b4b8e2c5-d5ef-4f95-8130-fe918dade8b6 kind: Event lastTimestamp: "2026-07-02T14:37:01Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:35:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:01Z" name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f.18be7fb46d3999ce namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: 4abd6b81-d712-4c7b-aa23-21b8b0879a93 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f namespace: kserve-ci-e2e-test resourceVersion: "19570" uid: b4b8e2c5-d5ef-4f95-8130-fe918dade8b6 kind: Event lastTimestamp: "2026-07-02T14:36:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:36:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:58Z" name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f.18be7fcc7dbac98a namespace: kserve-ci-e2e-test resourceVersion: "20499" uid: 64146462-8b95-4e39-8478-0449d8f85183 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f namespace: kserve-ci-e2e-test resourceVersion: "19570" uid: b4b8e2c5-d5ef-4f95-8130-fe918dade8b6 kind: Event lastTimestamp: "2026-07-02T14:36:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:36:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:58Z" name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f.18be7fcc7dbbfc79 namespace: kserve-ci-e2e-test resourceVersion: "20500" uid: 498d0ef1-f5e8-425b-b465-3ef9c507a715 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f namespace: kserve-ci-e2e-test resourceVersion: "19570" uid: b4b8e2c5-d5ef-4f95-8130-fe918dade8b6 kind: Event lastTimestamp: "2026-07-02T14:37:01Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:01Z" name: isvc-primary-571456-predictor-bdc8f5d66-rdw9f.18be7fcd1b78a205 namespace: kserve-ci-e2e-test resourceVersion: "20524" uid: c03d1f0d-b3d5-4763-b8de-5bb244d1cc3b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:35:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-571456-predictor-bdc8f5d66 namespace: kserve-ci-e2e-test resourceVersion: "19558" uid: d482bcec-e803-4eae-a9fc-4505f0e244f1 kind: Event lastTimestamp: "2026-07-02T14:35:08Z" message: 'Created pod: isvc-primary-571456-predictor-bdc8f5d66-rdw9f' metadata: creationTimestamp: "2026-07-02T14:35:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:35:08Z" name: isvc-primary-571456-predictor-bdc8f5d66.18be7fb2ce8f342b namespace: kserve-ci-e2e-test resourceVersion: "19569" uid: 43f36b32-d2af-40c4-92e4-68f50f4221f1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:35:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-571456-predictor namespace: kserve-ci-e2e-test resourceVersion: "19556" uid: 9b8ffb6e-52c4-4d21-a5e5-082e21957dae kind: Event lastTimestamp: "2026-07-02T14:35:08Z" message: Scaled up replica set isvc-primary-571456-predictor-bdc8f5d66 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:35:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:35:08Z" name: isvc-primary-571456-predictor.18be7fb2c9e202a1 namespace: kserve-ci-e2e-test resourceVersion: "19560" uid: 478e036f-99df-43af-a834-e42e8a69407f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:35:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-571456-predictor namespace: kserve-ci-e2e-test resourceVersion: "19557" uid: f965bd2f-1f02-46cc-a2b6-c65a81a53b47 kind: Event lastTimestamp: "2026-07-02T14:35:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:35:38Z" name: isvc-primary-571456-predictor.18be7fb64a0672f6 namespace: kserve-ci-e2e-test resourceVersion: "19823" uid: 95b9b8a3-d6c0-4f3f-8d20-9ed59e35161c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:35:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-571456-predictor namespace: kserve-ci-e2e-test resourceVersion: "19557" uid: f965bd2f-1f02-46cc-a2b6-c65a81a53b47 kind: Event lastTimestamp: "2026-07-02T14:35:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:35:38Z" name: isvc-primary-571456-predictor.18be7fb64b1a828a namespace: kserve-ci-e2e-test resourceVersion: "19824" uid: 68c53544-3881-4404-bf40-679f6fd0f24f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:35:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-571456-predictor namespace: kserve-ci-e2e-test resourceVersion: "19717" uid: f965bd2f-1f02-46cc-a2b6-c65a81a53b47 kind: Event lastTimestamp: "2026-07-02T14:36:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:35:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:36:53Z" name: isvc-primary-571456-predictor.18be7fbd48eb8c09 namespace: kserve-ci-e2e-test resourceVersion: "20427" uid: ea9da887-f7ac-4ef9-9cbb-248f1b612f71 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:35:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-571456-predictor namespace: kserve-ci-e2e-test resourceVersion: "19717" uid: f965bd2f-1f02-46cc-a2b6-c65a81a53b47 kind: Event lastTimestamp: "2026-07-02T14:36:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:35:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:36:53Z" name: isvc-primary-571456-predictor.18be7fbd49567646 namespace: kserve-ci-e2e-test resourceVersion: "20428" uid: 823a8442-2b5d-47eb-9f6a-10355f989fb6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T14:35:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-571456 namespace: kserve-ci-e2e-test resourceVersion: "19549" uid: f29e8048-b6c5-4abf-93a1-a6c8b50b8254 kind: Event lastTimestamp: "2026-07-02T14:36:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:35:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:36:31Z" name: isvc-primary-571456.18be7fb2c9f586a2 namespace: kserve-ci-e2e-test resourceVersion: "20199" uid: 3a28ea4e-52de-4ad1-80a0-47e4a2ea34df reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:35:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-571456 namespace: kserve-ci-e2e-test resourceVersion: "19549" uid: f29e8048-b6c5-4abf-93a1-a6c8b50b8254 kind: Event lastTimestamp: "2026-07-02T14:35:08Z" message: 'Failed to update status for InferenceService "isvc-primary-571456": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-571456": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:35:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:35:08Z" name: isvc-primary-571456.18be7fb2ca606f0b namespace: kserve-ci-e2e-test resourceVersion: "19579" uid: e2525572-b96a-41d6-9cec-79a1271d2a89 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:35:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-571456 namespace: kserve-ci-e2e-test resourceVersion: "19549" uid: f29e8048-b6c5-4abf-93a1-a6c8b50b8254 kind: Event lastTimestamp: "2026-07-02T14:35:08Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-571456": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:35:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:35:08Z" name: isvc-primary-571456.18be7fb2ca60bc0b namespace: kserve-ci-e2e-test resourceVersion: "19580" uid: a1d2959b-4e96-479e-a60c-f6aab85a4f2a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-571456 namespace: kserve-ci-e2e-test resourceVersion: "20197" uid: f29e8048-b6c5-4abf-93a1-a6c8b50b8254 kind: Event lastTimestamp: "2026-07-02T14:36:31Z" message: InferenceService [isvc-primary-571456] is Ready metadata: creationTimestamp: "2026-07-02T14:36:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:36:31Z" name: isvc-primary-571456.18be7fc62485f772 namespace: kserve-ci-e2e-test resourceVersion: "20198" uid: 651f8e24-6de6-467d-8729-f550cce9281d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:25:01.947249Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn namespace: kserve-ci-e2e-test resourceVersion: "14376" uid: 7f759183-a980-4aa1-be4b-fd8f4f3b9222 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn to ip-10-0-132-82.ec2.internal metadata: creationTimestamp: "2026-07-02T14:25:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:25:01Z" name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn.18be7f2599452764 namespace: kserve-ci-e2e-test resourceVersion: "14382" uid: f9254402-8d30-4c31-bc5e-d9d60e64270e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75fd45cf98-dpdjc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn namespace: kserve-ci-e2e-test resourceVersion: "14387" uid: 7f759183-a980-4aa1-be4b-fd8f4f3b9222 kind: Event lastTimestamp: "2026-07-02T14:25:02Z" message: Add eth0 [10.134.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:25:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:25:02Z" name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn.18be7f25b2919ee0 namespace: kserve-ci-e2e-test resourceVersion: "14396" uid: 4f34f735-8576-4b40-98ed-bfd1a85ea36e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 7f759183-a980-4aa1-be4b-fd8f4f3b9222 kind: Event lastTimestamp: "2026-07-02T14:25:02Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:8e2b89e7009febc315fad3ff8d06e5aec68b35b7cc22df79daf7c1334f909f5a" metadata: creationTimestamp: "2026-07-02T14:25:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:02Z" name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn.18be7f25b41a75fa namespace: kserve-ci-e2e-test resourceVersion: "14399" uid: 371c2403-ec8e-4b22-9679-c28420d0de10 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 7f759183-a980-4aa1-be4b-fd8f4f3b9222 kind: Event lastTimestamp: "2026-07-02T14:25:06Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:8e2b89e7009febc315fad3ff8d06e5aec68b35b7cc22df79daf7c1334f909f5a" in 3.7s (3.7s including waiting). Image size: 301859831 bytes.' metadata: creationTimestamp: "2026-07-02T14:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:06Z" name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn.18be7f2690a7dbf2 namespace: kserve-ci-e2e-test resourceVersion: "14426" uid: 5a50e404-0e73-4e6a-b3a2-ba6e3a172bd7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 7f759183-a980-4aa1-be4b-fd8f4f3b9222 kind: Event lastTimestamp: "2026-07-02T14:25:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:06Z" name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn.18be7f2693e4e8d2 namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: caed97e1-4918-48c4-8254-d6c640107032 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 7f759183-a980-4aa1-be4b-fd8f4f3b9222 kind: Event lastTimestamp: "2026-07-02T14:25:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:25:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:06Z" name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn.18be7f26945cc843 namespace: kserve-ci-e2e-test resourceVersion: "14428" uid: c0b07378-c978-4267-88c6-557dbfeda57c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 7f759183-a980-4aa1-be4b-fd8f4f3b9222 kind: Event lastTimestamp: "2026-07-02T14:25:10Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1697" metadata: creationTimestamp: "2026-07-02T14:25:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:10Z" name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn.18be7f2783af9395 namespace: kserve-ci-e2e-test resourceVersion: "14465" uid: 66b28fbb-5954-40af-9d3f-c975bb8df9ba reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 7f759183-a980-4aa1-be4b-fd8f4f3b9222 kind: Event lastTimestamp: "2026-07-02T14:25:24Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1697" in 14.349s (14.349s including waiting). Image size: 1555893189 bytes.' metadata: creationTimestamp: "2026-07-02T14:25:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:24Z" name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn.18be7f2adaff3bf2 namespace: kserve-ci-e2e-test resourceVersion: "14568" uid: 4b91626e-4310-42aa-bc6b-1a8c6383d29d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 7f759183-a980-4aa1-be4b-fd8f4f3b9222 kind: Event lastTimestamp: "2026-07-02T14:25:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:25:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:24Z" name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn.18be7f2addf90547 namespace: kserve-ci-e2e-test resourceVersion: "14569" uid: cf3ab5f1-b380-464b-b643-0296271ddbdd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 7f759183-a980-4aa1-be4b-fd8f4f3b9222 kind: Event lastTimestamp: "2026-07-02T14:25:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:25:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:24Z" name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn.18be7f2ade67d61d namespace: kserve-ci-e2e-test resourceVersion: "14570" uid: 2d5667e4-bd71-440c-9d1c-ea0699eb935d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 7f759183-a980-4aa1-be4b-fd8f4f3b9222 kind: Event lastTimestamp: "2026-07-02T14:25:24Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T14:25:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:24Z" name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn.18be7f2ade757ad0 namespace: kserve-ci-e2e-test resourceVersion: "14571" uid: f657fcf4-99c6-422b-a57e-1450271072e8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 7f759183-a980-4aa1-be4b-fd8f4f3b9222 kind: Event lastTimestamp: "2026-07-02T14:25:27Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.961s (2.961s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T14:25:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:27Z" name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn.18be7f2b8ef7f610 namespace: kserve-ci-e2e-test resourceVersion: "14590" uid: ba41d61e-78cb-4e1c-8441-c2a617b23371 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 7f759183-a980-4aa1-be4b-fd8f4f3b9222 kind: Event lastTimestamp: "2026-07-02T14:25:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:25:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:27Z" name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn.18be7f2b91c56c11 namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: 5ad487e8-9b9d-4d3e-ba43-9b8e24e7c829 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 7f759183-a980-4aa1-be4b-fd8f4f3b9222 kind: Event lastTimestamp: "2026-07-02T14:25:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:25:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:27Z" name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn.18be7f2b92539719 namespace: kserve-ci-e2e-test resourceVersion: "14594" uid: 2044a6c9-0e5f-4220-96f8-472b40ceb356 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 7f759183-a980-4aa1-be4b-fd8f4f3b9222 kind: Event lastTimestamp: "2026-07-02T14:25:27Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:e129573a9a25e59fc4e926ae207b9073f8d71d58bb61a455387cef8a5c19e72b" metadata: creationTimestamp: "2026-07-02T14:25:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:27Z" name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn.18be7f2b925f47f3 namespace: kserve-ci-e2e-test resourceVersion: "14595" uid: b48bc273-7277-4f7e-85bc-bdae63344d0c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 7f759183-a980-4aa1-be4b-fd8f4f3b9222 kind: Event lastTimestamp: "2026-07-02T14:25:31Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:e129573a9a25e59fc4e926ae207b9073f8d71d58bb61a455387cef8a5c19e72b" in 3.467s (3.467s including waiting). Image size: 239994522 bytes.' metadata: creationTimestamp: "2026-07-02T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:31Z" name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn.18be7f2c610af594 namespace: kserve-ci-e2e-test resourceVersion: "14619" uid: e92dea36-5a59-4161-9f00-3a516967dc2a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 7f759183-a980-4aa1-be4b-fd8f4f3b9222 kind: Event lastTimestamp: "2026-07-02T14:25:31Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-02T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:31Z" name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn.18be7f2c64233940 namespace: kserve-ci-e2e-test resourceVersion: "14620" uid: 064dc19a-f2b7-4b52-9721-35433a19f6dc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 7f759183-a980-4aa1-be4b-fd8f4f3b9222 kind: Event lastTimestamp: "2026-07-02T14:25:31Z" message: Started container agent metadata: creationTimestamp: "2026-07-02T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:31Z" name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn.18be7f2c64976cd3 namespace: kserve-ci-e2e-test resourceVersion: "14621" uid: 3e684233-8515-4dbb-90db-6fa0f9445f1a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T14:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 7f759183-a980-4aa1-be4b-fd8f4f3b9222 kind: Event lastTimestamp: "2026-07-02T14:27:06Z" message: 'Readiness probe failed: dial tcp 10.134.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:06Z" name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn.18be7f2c6cf3c3cb namespace: kserve-ci-e2e-test resourceVersion: "15446" uid: 4f6d4e0f-fd82-4377-8ad8-86586255b531 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T14:25:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 7f759183-a980-4aa1-be4b-fd8f4f3b9222 kind: Event lastTimestamp: "2026-07-02T14:27:06Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:25:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:06Z" name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn.18be7f2ca8d124c0 namespace: kserve-ci-e2e-test resourceVersion: "15447" uid: 1dd6ec2e-3fd1-47dc-8a4d-b41a1d12664c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 7f759183-a980-4aa1-be4b-fd8f4f3b9222 kind: Event lastTimestamp: "2026-07-02T14:26:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:57Z" name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn.18be7f406c7d6186 namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: b1a031ea-c944-4be4-bbe9-793d135b0a74 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 7f759183-a980-4aa1-be4b-fd8f4f3b9222 kind: Event lastTimestamp: "2026-07-02T14:26:57Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:57Z" name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn.18be7f406c7e18ac namespace: kserve-ci-e2e-test resourceVersion: "15289" uid: 28cf3b16-6da8-41a7-8c69-f530f172c8a4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 7f759183-a980-4aa1-be4b-fd8f4f3b9222 kind: Event lastTimestamp: "2026-07-02T14:26:57Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-02T14:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:57Z" name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn.18be7f406c7fbee6 namespace: kserve-ci-e2e-test resourceVersion: "15290" uid: 918f70dd-4575-49fe-8f10-f70448a37e1e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:26:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 7f759183-a980-4aa1-be4b-fd8f4f3b9222 kind: Event lastTimestamp: "2026-07-02T14:27:02Z" message: 'Readiness probe failed: Get "https://10.134.0.18:8643/healthz": dial tcp 10.134.0.18:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:02Z" name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn.18be7f4073337a2b namespace: kserve-ci-e2e-test resourceVersion: "15415" uid: 09c34e6d-a6d8-4cef-8686-9ae00dc2b74e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff namespace: kserve-ci-e2e-test resourceVersion: "14367" uid: 5a1af8c7-2a66-40d2-affa-8a6cdb386ffc kind: Event lastTimestamp: "2026-07-02T14:25:01Z" message: 'Created pod: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff-7dpfn' metadata: creationTimestamp: "2026-07-02T14:25:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:25:01Z" name: isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff.18be7f259897d474 namespace: kserve-ci-e2e-test resourceVersion: "14377" uid: 6485abaa-6eb0-40b0-a776-708cca522f30 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-a23ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "14366" uid: aff68ec2-c316-459d-8149-a21005bc11d5 kind: Event lastTimestamp: "2026-07-02T14:25:01Z" message: Scaled up replica set isvc-raw-sklearn-batcher-a23ad-predictor-9694547ff from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:25:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:25:01Z" name: isvc-raw-sklearn-batcher-a23ad-predictor.18be7f25939a0084 namespace: kserve-ci-e2e-test resourceVersion: "14368" uid: b25c6d27-783f-4e34-b1fd-2cced985fca4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:25:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-a23ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "14380" uid: baeeca68-3d7c-4e82-aea7-2d1e4aad976f kind: Event lastTimestamp: "2026-07-02T14:25:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:25:47Z" name: isvc-raw-sklearn-batcher-a23ad-predictor.18be7f291934d7b9 namespace: kserve-ci-e2e-test resourceVersion: "14745" uid: 1f45a936-0664-4c39-b1cb-c8f7e0ed2d4b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:25:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-a23ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "14380" uid: baeeca68-3d7c-4e82-aea7-2d1e4aad976f kind: Event lastTimestamp: "2026-07-02T14:25:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:25:47Z" name: isvc-raw-sklearn-batcher-a23ad-predictor.18be7f2919a23c3d namespace: kserve-ci-e2e-test resourceVersion: "14746" uid: 3793e16f-7044-436b-8333-b29045460fa2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:26:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-a23ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: baeeca68-3d7c-4e82-aea7-2d1e4aad976f kind: Event lastTimestamp: "2026-07-02T14:26:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:26:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:26:47Z" name: isvc-raw-sklearn-batcher-a23ad-predictor.18be7f3396ef0441 namespace: kserve-ci-e2e-test resourceVersion: "15179" uid: d7ef6d7c-9de5-4758-b3c8-5bb036c649b8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:26:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-a23ad-predictor namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: baeeca68-3d7c-4e82-aea7-2d1e4aad976f kind: Event lastTimestamp: "2026-07-02T14:26:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:26:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:26:47Z" name: isvc-raw-sklearn-batcher-a23ad-predictor.18be7f339759b9ae namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: f24dda62-c405-429f-87b4-1f3ffefcf83d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T14:25:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-a23ad namespace: kserve-ci-e2e-test resourceVersion: "14357" uid: 0553d9f5-26c2-4a1e-b923-68f7dad5e777 kind: Event lastTimestamp: "2026-07-02T14:26:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:25:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:26:46Z" name: isvc-raw-sklearn-batcher-a23ad.18be7f2599ebca01 namespace: kserve-ci-e2e-test resourceVersion: "15174" uid: fad01f77-68e7-478e-9b00-bd4e3e230e82 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-a23ad namespace: kserve-ci-e2e-test resourceVersion: "14357" uid: 0553d9f5-26c2-4a1e-b923-68f7dad5e777 kind: Event lastTimestamp: "2026-07-02T14:25:01Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-a23ad": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-a23ad": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:25:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:25:01Z" name: isvc-raw-sklearn-batcher-a23ad.18be7f259a5915c9 namespace: kserve-ci-e2e-test resourceVersion: "14386" uid: 1744a700-5d70-4e29-a730-dd3d7324e39e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-a23ad namespace: kserve-ci-e2e-test resourceVersion: "14357" uid: 0553d9f5-26c2-4a1e-b923-68f7dad5e777 kind: Event lastTimestamp: "2026-07-02T14:25:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-a23ad": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:25:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:25:01Z" name: isvc-raw-sklearn-batcher-a23ad.18be7f259a5978f3 namespace: kserve-ci-e2e-test resourceVersion: "14388" uid: 1fa75f1c-fc92-40ad-94fb-919cb475cd4c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-a23ad namespace: kserve-ci-e2e-test resourceVersion: "15172" uid: 0553d9f5-26c2-4a1e-b923-68f7dad5e777 kind: Event lastTimestamp: "2026-07-02T14:26:46Z" message: InferenceService [isvc-raw-sklearn-batcher-a23ad] is Ready metadata: creationTimestamp: "2026-07-02T14:26:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:26:46Z" name: isvc-raw-sklearn-batcher-a23ad.18be7f3de31776d9 namespace: kserve-ci-e2e-test resourceVersion: "15173" uid: aeac0fa2-7d42-486a-9d7a-278522af18f6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:36:38.659555Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-571456-predictor-7b4f79889c-pdzpt namespace: kserve-ci-e2e-test resourceVersion: "20280" uid: 22c7c681-ac3c-4b3e-90ac-ef27d37fe65e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-571456-predictor-7b4f79889c-pdzpt to ip-10-0-132-82.ec2.internal metadata: creationTimestamp: "2026-07-02T14:36:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:36:38Z" name: isvc-secondary-571456-predictor-7b4f79889c-pdzpt.18be7fc7d08f44a1 namespace: kserve-ci-e2e-test resourceVersion: "20285" uid: 70344e8d-c4b9-44ba-a57f-c429b509dd50 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75fd45cf98-dpdjc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:38Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-571456-predictor-7b4f79889c-pdzpt namespace: kserve-ci-e2e-test resourceVersion: "20282" uid: 22c7c681-ac3c-4b3e-90ac-ef27d37fe65e kind: Event lastTimestamp: "2026-07-02T14:36:38Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-secondary-571456-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:36:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:38Z" name: isvc-secondary-571456-predictor-7b4f79889c-pdzpt.18be7fc7d735b667 namespace: kserve-ci-e2e-test resourceVersion: "20296" uid: 5b5a2924-c27a-4c90-b8c1-c25a9f745aee reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-571456-predictor-7b4f79889c-pdzpt namespace: kserve-ci-e2e-test resourceVersion: "20291" uid: 22c7c681-ac3c-4b3e-90ac-ef27d37fe65e kind: Event lastTimestamp: "2026-07-02T14:36:39Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:36:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:36:39Z" name: isvc-secondary-571456-predictor-7b4f79889c-pdzpt.18be7fc80dd941f0 namespace: kserve-ci-e2e-test resourceVersion: "20307" uid: afcd5e05-e6e1-4b4b-a077-79d5b6f80e69 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:36:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-571456-predictor-7b4f79889c-pdzpt namespace: kserve-ci-e2e-test resourceVersion: "20282" uid: 22c7c681-ac3c-4b3e-90ac-ef27d37fe65e kind: Event lastTimestamp: "2026-07-02T14:36:46Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8e2b89e7009febc315fad3ff8d06e5aec68b35b7cc22df79daf7c1334f909f5a" already present on machine metadata: creationTimestamp: "2026-07-02T14:36:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:46Z" name: isvc-secondary-571456-predictor-7b4f79889c-pdzpt.18be7fc80f6c18c8 namespace: kserve-ci-e2e-test resourceVersion: "20367" uid: 981135f1-65e0-4edb-b9fd-e279ef499be2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:36:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-571456-predictor-7b4f79889c-pdzpt namespace: kserve-ci-e2e-test resourceVersion: "20282" uid: 22c7c681-ac3c-4b3e-90ac-ef27d37fe65e kind: Event lastTimestamp: "2026-07-02T14:36:46Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:36:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:46Z" name: isvc-secondary-571456-predictor-7b4f79889c-pdzpt.18be7fc8129939ca namespace: kserve-ci-e2e-test resourceVersion: "20369" uid: 3867098b-5927-4489-b758-6e7d3fa23c1a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:36:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-571456-predictor-7b4f79889c-pdzpt namespace: kserve-ci-e2e-test resourceVersion: "20282" uid: 22c7c681-ac3c-4b3e-90ac-ef27d37fe65e kind: Event lastTimestamp: "2026-07-02T14:36:46Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:36:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:46Z" name: isvc-secondary-571456-predictor-7b4f79889c-pdzpt.18be7fc81308caa4 namespace: kserve-ci-e2e-test resourceVersion: "20371" uid: af717f37-74f2-43d5-862f-4b004b41d408 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-571456-predictor-7b4f79889c-pdzpt namespace: kserve-ci-e2e-test resourceVersion: "20282" uid: 22c7c681-ac3c-4b3e-90ac-ef27d37fe65e kind: Event lastTimestamp: "2026-07-02T14:36:50Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-571456-predictor-7b4f79889c-pdzpt_kserve-ci-e2e-test(22c7c681-ac3c-4b3e-90ac-ef27d37fe65e) metadata: creationTimestamp: "2026-07-02T14:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:50Z" name: isvc-secondary-571456-predictor-7b4f79889c-pdzpt.18be7fcaa05b2846 namespace: kserve-ci-e2e-test resourceVersion: "20405" uid: e16029d0-8b67-4902-bc5d-1ef7e4976598 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-571456-predictor-7b4f79889c namespace: kserve-ci-e2e-test resourceVersion: "20270" uid: f9f310aa-e802-423c-8606-7732031873de kind: Event lastTimestamp: "2026-07-02T14:36:38Z" message: 'Created pod: isvc-secondary-571456-predictor-7b4f79889c-pdzpt' metadata: creationTimestamp: "2026-07-02T14:36:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:36:38Z" name: isvc-secondary-571456-predictor-7b4f79889c.18be7fc7d01c459f namespace: kserve-ci-e2e-test resourceVersion: "20281" uid: e8642650-1da7-40c6-a508-2dd545d6040c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-571456-predictor namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: 303b256c-9452-4214-8494-40f4d36e36bd kind: Event lastTimestamp: "2026-07-02T14:36:38Z" message: Scaled up replica set isvc-secondary-571456-predictor-7b4f79889c from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:36:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:36:38Z" name: isvc-secondary-571456-predictor.18be7fc7cb05920d namespace: kserve-ci-e2e-test resourceVersion: "20271" uid: 45b7ad97-c393-4ad5-bf65-0f17f8736f7f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-571456-predictor namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 435ee66f-1043-45b1-b187-22ff4c827032 kind: Event lastTimestamp: "2026-07-02T14:36:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:36:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:36:53Z" name: isvc-secondary-571456-predictor.18be7fcb498ffdc6 namespace: kserve-ci-e2e-test resourceVersion: "20429" uid: 703e1e0d-9dda-4025-92c7-2a5f68c0ca54 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-571456-predictor namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 435ee66f-1043-45b1-b187-22ff4c827032 kind: Event lastTimestamp: "2026-07-02T14:36:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:36:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:36:53Z" name: isvc-secondary-571456-predictor.18be7fcb4a0c669c namespace: kserve-ci-e2e-test resourceVersion: "20431" uid: 2f57dfa2-a089-4b4c-ac80-0a642bfefefe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-07-02T14:36:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-571456 namespace: kserve-ci-e2e-test resourceVersion: "20261" uid: 58743c20-d2ad-4c1f-a051-e95952a96bfa kind: Event lastTimestamp: "2026-07-02T14:36:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:36:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:36:50Z" name: isvc-secondary-571456.18be7fc7cb088ff1 namespace: kserve-ci-e2e-test resourceVersion: "20412" uid: 058bfa55-e381-44e9-8bdb-577cf7a912ed reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:36:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-571456 namespace: kserve-ci-e2e-test resourceVersion: "20261" uid: 58743c20-d2ad-4c1f-a051-e95952a96bfa kind: Event lastTimestamp: "2026-07-02T14:36:38Z" message: 'Failed to update status for InferenceService "isvc-secondary-571456": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-571456": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:36:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:36:38Z" name: isvc-secondary-571456.18be7fc7cba95c16 namespace: kserve-ci-e2e-test resourceVersion: "20290" uid: db311fba-16de-4bed-b9df-c12117b6a13f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:36:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-571456 namespace: kserve-ci-e2e-test resourceVersion: "20261" uid: 58743c20-d2ad-4c1f-a051-e95952a96bfa kind: Event lastTimestamp: "2026-07-02T14:36:38Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-571456": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:36:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:36:38Z" name: isvc-secondary-571456.18be7fc7cba99691 namespace: kserve-ci-e2e-test resourceVersion: "20292" uid: 0b4a6ca6-f2e2-4458-98aa-b632e0af879f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:26:57.208233Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt namespace: kserve-ci-e2e-test resourceVersion: "15293" uid: cbb193ee-77ed-40f8-9b0c-43372ec20ca7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt to ip-10-0-132-82.ec2.internal metadata: creationTimestamp: "2026-07-02T14:26:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:26:57Z" name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt.18be7f406f5c287a namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: 3b5e0303-7943-4707-bd3f-74a3910f0a61 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75fd45cf98-dpdjc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:57Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt namespace: kserve-ci-e2e-test resourceVersion: "15313" uid: cbb193ee-77ed-40f8-9b0c-43372ec20ca7 kind: Event lastTimestamp: "2026-07-02T14:26:57Z" message: Add eth0 [10.134.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:26:57Z" name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt.18be7f4088722600 namespace: kserve-ci-e2e-test resourceVersion: "15347" uid: f9fe5b49-a868-4351-931a-35566c4c3f3e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: cbb193ee-77ed-40f8-9b0c-43372ec20ca7 kind: Event lastTimestamp: "2026-07-02T14:26:57Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8e2b89e7009febc315fad3ff8d06e5aec68b35b7cc22df79daf7c1334f909f5a" already present on machine metadata: creationTimestamp: "2026-07-02T14:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:57Z" name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt.18be7f4089ee2513 namespace: kserve-ci-e2e-test resourceVersion: "15350" uid: f541b707-7fef-48b6-828e-458fe1918a42 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: cbb193ee-77ed-40f8-9b0c-43372ec20ca7 kind: Event lastTimestamp: "2026-07-02T14:26:57Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:57Z" name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt.18be7f408cc53f59 namespace: kserve-ci-e2e-test resourceVersion: "15351" uid: 0431bc23-cea6-4e56-81d2-5151ae85b347 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: cbb193ee-77ed-40f8-9b0c-43372ec20ca7 kind: Event lastTimestamp: "2026-07-02T14:26:57Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:57Z" name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt.18be7f408d32aea8 namespace: kserve-ci-e2e-test resourceVersion: "15352" uid: 4e38f753-0927-49b3-98eb-c2b9d8964b8e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: cbb193ee-77ed-40f8-9b0c-43372ec20ca7 kind: Event lastTimestamp: "2026-07-02T14:27:01Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:01Z" name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt.18be7f4174be6712 namespace: kserve-ci-e2e-test resourceVersion: "15399" uid: 1ac9aa0b-53da-4073-855b-ac77bf524766 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: cbb193ee-77ed-40f8-9b0c-43372ec20ca7 kind: Event lastTimestamp: "2026-07-02T14:27:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:01Z" name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt.18be7f417bebfdcf namespace: kserve-ci-e2e-test resourceVersion: "15405" uid: 7c5a26e0-f57f-4d91-a077-77cb022a8251 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: cbb193ee-77ed-40f8-9b0c-43372ec20ca7 kind: Event lastTimestamp: "2026-07-02T14:27:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:01Z" name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt.18be7f417c5bcb19 namespace: kserve-ci-e2e-test resourceVersion: "15406" uid: 72bbdc63-b256-4fc9-9732-d5f60adc7d19 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: cbb193ee-77ed-40f8-9b0c-43372ec20ca7 kind: Event lastTimestamp: "2026-07-02T14:27:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:01Z" name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt.18be7f417c6640c3 namespace: kserve-ci-e2e-test resourceVersion: "15407" uid: 994908bc-dfbc-4137-9165-d23c544ea5d7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: cbb193ee-77ed-40f8-9b0c-43372ec20ca7 kind: Event lastTimestamp: "2026-07-02T14:27:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:01Z" name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt.18be7f417fc2fdad namespace: kserve-ci-e2e-test resourceVersion: "15409" uid: d1b14b71-3814-4c6e-9854-d43d60a4e727 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: cbb193ee-77ed-40f8-9b0c-43372ec20ca7 kind: Event lastTimestamp: "2026-07-02T14:27:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:01Z" name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt.18be7f4180391442 namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 99b7d7bb-11a9-46c0-963e-75b37e749ef0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T14:27:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: cbb193ee-77ed-40f8-9b0c-43372ec20ca7 kind: Event lastTimestamp: "2026-07-02T14:28:48Z" message: 'Readiness probe failed: dial tcp 10.134.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:27:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:48Z" name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt.18be7f41b0fe37a8 namespace: kserve-ci-e2e-test resourceVersion: "16395" uid: a6bbec29-55b0-4704-b9d7-a79ce56bb254 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: cbb193ee-77ed-40f8-9b0c-43372ec20ca7 kind: Event lastTimestamp: "2026-07-02T14:28:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:47Z" name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt.18be7f5a1edb9be6 namespace: kserve-ci-e2e-test resourceVersion: "16313" uid: db1c1651-743f-4f64-9973-91942c04ff90 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: cbb193ee-77ed-40f8-9b0c-43372ec20ca7 kind: Event lastTimestamp: "2026-07-02T14:28:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:47Z" name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt.18be7f5a1edc335d namespace: kserve-ci-e2e-test resourceVersion: "16318" uid: 8de39879-f26b-48c4-8319-be3935712888 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: cbb193ee-77ed-40f8-9b0c-43372ec20ca7 kind: Event lastTimestamp: "2026-07-02T14:28:48Z" message: 'Readiness probe failed: Get "https://10.134.0.19:8643/healthz": dial tcp 10.134.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:28:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:48Z" name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt.18be7f5a5f58f538 namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: 8b0852d1-4961-425d-b85e-1104152e5627 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f namespace: kserve-ci-e2e-test resourceVersion: "15263" uid: 354f448a-6b2f-4b1c-bb87-1d534fc02f8d kind: Event lastTimestamp: "2026-07-02T14:26:57Z" message: 'Created pod: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f-r7hdt' metadata: creationTimestamp: "2026-07-02T14:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:26:57Z" name: isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f.18be7f406e9af758 namespace: kserve-ci-e2e-test resourceVersion: "15297" uid: b0dc7f96-dcb9-4a35-80e1-d12304b258fc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-042a3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15261" uid: 98d8cf2f-eaca-4abd-a19d-ce841bfdbd7b kind: Event lastTimestamp: "2026-07-02T14:26:57Z" message: Scaled up replica set isvc-sklearn-graph-raw-042a3-predictor-c6d85fb5f from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:26:57Z" name: isvc-sklearn-graph-raw-042a3-predictor.18be7f40698ae85f namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: 52e03887-4b90-443a-8a90-07e8849a48af reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:27:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-042a3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15262" uid: 342a64d8-07fb-4f9d-9ae7-a1a812d5e3c5 kind: Event lastTimestamp: "2026-07-02T14:27:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:42Z" name: isvc-sklearn-graph-raw-042a3-predictor.18be7f43e7fd43dc namespace: kserve-ci-e2e-test resourceVersion: "15722" uid: 2269cb28-db94-4f01-9491-86051d270ea6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:27:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-042a3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15262" uid: 342a64d8-07fb-4f9d-9ae7-a1a812d5e3c5 kind: Event lastTimestamp: "2026-07-02T14:27:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:42Z" name: isvc-sklearn-graph-raw-042a3-predictor.18be7f43e86457c5 namespace: kserve-ci-e2e-test resourceVersion: "15723" uid: c9b3d302-099d-4457-85a3-8a9abffbc2d4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:27:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-042a3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15490" uid: 342a64d8-07fb-4f9d-9ae7-a1a812d5e3c5 kind: Event lastTimestamp: "2026-07-02T14:28:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:42Z" name: isvc-sklearn-graph-raw-042a3-predictor.18be7f476779e279 namespace: kserve-ci-e2e-test resourceVersion: "16213" uid: 4eac9d73-9200-4aae-8103-83f2aac783ae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:27:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-042a3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15490" uid: 342a64d8-07fb-4f9d-9ae7-a1a812d5e3c5 kind: Event lastTimestamp: "2026-07-02T14:28:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:42Z" name: isvc-sklearn-graph-raw-042a3-predictor.18be7f4767dcfb05 namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: d2da3b1d-6e24-40b9-bd45-183c35fd4bf9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T14:26:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-042a3 namespace: kserve-ci-e2e-test resourceVersion: "15254" uid: e9272a5d-0c06-4876-965a-1ebe66920c0e kind: Event lastTimestamp: "2026-07-02T14:28:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:28:18Z" name: isvc-sklearn-graph-raw-042a3.18be7f40698c222e namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: 3bde39b5-72be-4964-aade-7842cd07fe99 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-042a3 namespace: kserve-ci-e2e-test resourceVersion: "15999" uid: e9272a5d-0c06-4876-965a-1ebe66920c0e kind: Event lastTimestamp: "2026-07-02T14:28:18Z" message: InferenceService [isvc-sklearn-graph-raw-042a3] is Ready metadata: creationTimestamp: "2026-07-02T14:28:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:28:18Z" name: isvc-sklearn-graph-raw-042a3.18be7f5368cef23c namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: 0321ec6e-8c35-4bba-92ae-973aa0654fc5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:28:47.527120Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk namespace: kserve-ci-e2e-test resourceVersion: "16304" uid: 5a623e37-311c-4b1d-a32e-d7c8ea7db99f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk to ip-10-0-132-82.ec2.internal metadata: creationTimestamp: "2026-07-02T14:28:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:28:47Z" name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk.18be7f5a1ee0c6cb namespace: kserve-ci-e2e-test resourceVersion: "16314" uid: 92da1606-1489-4fd3-98b5-6608eade490e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75fd45cf98-dpdjc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:47Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: 5a623e37-311c-4b1d-a32e-d7c8ea7db99f kind: Event lastTimestamp: "2026-07-02T14:28:47Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-raw-hpa-130af-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:47Z" name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk.18be7f5a25c9e308 namespace: kserve-ci-e2e-test resourceVersion: "16361" uid: 4d9bdaab-1838-49ce-8384-d2af53bba2cc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:48Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk namespace: kserve-ci-e2e-test resourceVersion: "16329" uid: 5a623e37-311c-4b1d-a32e-d7c8ea7db99f kind: Event lastTimestamp: "2026-07-02T14:28:48Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:28:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:28:48Z" name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk.18be7f5a4abcd146 namespace: kserve-ci-e2e-test resourceVersion: "16386" uid: 0defeffa-c972-4b14-93c5-6ec3aa02791e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: 5a623e37-311c-4b1d-a32e-d7c8ea7db99f kind: Event lastTimestamp: "2026-07-02T14:28:48Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8e2b89e7009febc315fad3ff8d06e5aec68b35b7cc22df79daf7c1334f909f5a" already present on machine metadata: creationTimestamp: "2026-07-02T14:28:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:48Z" name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk.18be7f5a4c1826c5 namespace: kserve-ci-e2e-test resourceVersion: "16388" uid: 2a501f20-8acd-4592-b8f3-56f0346a9915 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: 5a623e37-311c-4b1d-a32e-d7c8ea7db99f kind: Event lastTimestamp: "2026-07-02T14:28:48Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:28:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:48Z" name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk.18be7f5a4e964c41 namespace: kserve-ci-e2e-test resourceVersion: "16390" uid: ffd86a93-2448-4388-bcb3-6d3315147297 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: 5a623e37-311c-4b1d-a32e-d7c8ea7db99f kind: Event lastTimestamp: "2026-07-02T14:28:48Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:28:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:48Z" name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk.18be7f5a4f05bd02 namespace: kserve-ci-e2e-test resourceVersion: "16391" uid: fe190bf1-ce98-4ad1-a5b5-5ae2bc69ee42 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: 5a623e37-311c-4b1d-a32e-d7c8ea7db99f kind: Event lastTimestamp: "2026-07-02T14:28:52Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:28:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:52Z" name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk.18be7f5b64e26ccd namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: 4f240ca3-bf5b-49b0-9870-8c69045f9245 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: 5a623e37-311c-4b1d-a32e-d7c8ea7db99f kind: Event lastTimestamp: "2026-07-02T14:28:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:53Z" name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk.18be7f5b67f03e8e namespace: kserve-ci-e2e-test resourceVersion: "16456" uid: 5dae931c-1706-4402-8faa-ddc2c681fa79 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: 5a623e37-311c-4b1d-a32e-d7c8ea7db99f kind: Event lastTimestamp: "2026-07-02T14:28:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:53Z" name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk.18be7f5b685ef1ee namespace: kserve-ci-e2e-test resourceVersion: "16457" uid: 0f206614-8c4b-41a5-8cd3-95b18a026bdf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: 5a623e37-311c-4b1d-a32e-d7c8ea7db99f kind: Event lastTimestamp: "2026-07-02T14:28:53Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:53Z" name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk.18be7f5b68696023 namespace: kserve-ci-e2e-test resourceVersion: "16460" uid: 7a862190-f648-4169-8570-319dfa5ff119 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: 5a623e37-311c-4b1d-a32e-d7c8ea7db99f kind: Event lastTimestamp: "2026-07-02T14:28:53Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:53Z" name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk.18be7f5b6b4da2db namespace: kserve-ci-e2e-test resourceVersion: "16462" uid: a6020e4c-7162-476f-9289-ba053044099d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: 5a623e37-311c-4b1d-a32e-d7c8ea7db99f kind: Event lastTimestamp: "2026-07-02T14:28:53Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:53Z" name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk.18be7f5b6bc0e5a1 namespace: kserve-ci-e2e-test resourceVersion: "16463" uid: 01944146-f574-46c9-bc5a-7d8d0aa04e7f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T14:28:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: 5a623e37-311c-4b1d-a32e-d7c8ea7db99f kind: Event lastTimestamp: "2026-07-02T14:30:39Z" message: 'Readiness probe failed: dial tcp 10.134.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:28:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:39Z" name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk.18be7f5bdcb494e7 namespace: kserve-ci-e2e-test resourceVersion: "17393" uid: d15a6e22-722c-4198-bb57-17a3c73f068c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: 5a623e37-311c-4b1d-a32e-d7c8ea7db99f kind: Event lastTimestamp: "2026-07-02T14:30:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:30:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:37Z" name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk.18be7f73cfaa3da5 namespace: kserve-ci-e2e-test resourceVersion: "17355" uid: dbbfd8d8-38ce-4538-a1c0-df2a46cbd318 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: 5a623e37-311c-4b1d-a32e-d7c8ea7db99f kind: Event lastTimestamp: "2026-07-02T14:30:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:30:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:37Z" name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk.18be7f73cfaabee4 namespace: kserve-ci-e2e-test resourceVersion: "17357" uid: a0bff416-3261-4a9e-be52-187e2d46862d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: 5a623e37-311c-4b1d-a32e-d7c8ea7db99f kind: Event lastTimestamp: "2026-07-02T14:30:41Z" message: 'Readiness probe failed: Get "https://10.134.0.20:8643/healthz": dial tcp 10.134.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:30:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:41Z" name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk.18be7f748afc05bd namespace: kserve-ci-e2e-test resourceVersion: "17420" uid: c9d52c47-918c-4c1c-8908-08f97be7a613 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467 namespace: kserve-ci-e2e-test resourceVersion: "16281" uid: 5450703b-ecd2-4efb-8f4d-b43b6ea9e42b kind: Event lastTimestamp: "2026-07-02T14:28:47Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467-6wprk' metadata: creationTimestamp: "2026-07-02T14:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:47Z" name: isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467.18be7f5a1e40b1ac namespace: kserve-ci-e2e-test resourceVersion: "16308" uid: 230f1a9f-812a-4624-895f-cf9d708c0d10 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-130af-predictor namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: 5bb45c24-47c4-4bd3-87aa-ed711693142a kind: Event lastTimestamp: "2026-07-02T14:28:47Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-130af-predictor-6bd56fd467 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:47Z" name: isvc-sklearn-graph-raw-hpa-130af-predictor.18be7f5a199f02ee namespace: kserve-ci-e2e-test resourceVersion: "16282" uid: 41bcbd50-a8f3-421e-8017-06dfd2d449d4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:29:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-130af-predictor namespace: kserve-ci-e2e-test resourceVersion: "16280" uid: 60e38c87-39f2-4349-a975-bb72da00361a kind: Event lastTimestamp: "2026-07-02T14:29:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:29:17Z" name: isvc-sklearn-graph-raw-hpa-130af-predictor.18be7f5d981d0780 namespace: kserve-ci-e2e-test resourceVersion: "16657" uid: 1d67c74b-db52-40b5-93ee-4ede6625cf8a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:29:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-130af-predictor namespace: kserve-ci-e2e-test resourceVersion: "16280" uid: 60e38c87-39f2-4349-a975-bb72da00361a kind: Event lastTimestamp: "2026-07-02T14:29:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:29:17Z" name: isvc-sklearn-graph-raw-hpa-130af-predictor.18be7f5d98819f43 namespace: kserve-ci-e2e-test resourceVersion: "16658" uid: 89cb19e1-8845-475a-8bfe-3cb6c9b4f760 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:29:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-130af-predictor namespace: kserve-ci-e2e-test resourceVersion: "16551" uid: 60e38c87-39f2-4349-a975-bb72da00361a kind: Event lastTimestamp: "2026-07-02T14:30:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:29:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:30:17Z" name: isvc-sklearn-graph-raw-hpa-130af-predictor.18be7f649672c3e6 namespace: kserve-ci-e2e-test resourceVersion: "17105" uid: 73a0979c-01c3-485c-9498-dc66b9040fb0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:29:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-130af-predictor namespace: kserve-ci-e2e-test resourceVersion: "16551" uid: 60e38c87-39f2-4349-a975-bb72da00361a kind: Event lastTimestamp: "2026-07-02T14:30:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:29:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:30:17Z" name: isvc-sklearn-graph-raw-hpa-130af-predictor.18be7f6496dfb77b namespace: kserve-ci-e2e-test resourceVersion: "17106" uid: 345124b3-582e-4c06-a404-869776264f37 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T14:28:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-130af namespace: kserve-ci-e2e-test resourceVersion: "16270" uid: aaed20a8-773e-4b4d-ad91-b24002234c90 kind: Event lastTimestamp: "2026-07-02T14:30:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:30:09Z" name: isvc-sklearn-graph-raw-hpa-130af.18be7f5a19a46af0 namespace: kserve-ci-e2e-test resourceVersion: "17050" uid: f7b5f594-a23f-4ddb-ac6f-82073615d17a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-130af namespace: kserve-ci-e2e-test resourceVersion: "16270" uid: aaed20a8-773e-4b4d-ad91-b24002234c90 kind: Event lastTimestamp: "2026-07-02T14:28:47Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-130af": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-130af": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:28:47Z" name: isvc-sklearn-graph-raw-hpa-130af.18be7f5a1a232264 namespace: kserve-ci-e2e-test resourceVersion: "16285" uid: c92a5a2a-772c-4bf7-9394-8221b6a694fa reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-130af namespace: kserve-ci-e2e-test resourceVersion: "17048" uid: aaed20a8-773e-4b4d-ad91-b24002234c90 kind: Event lastTimestamp: "2026-07-02T14:30:09Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-130af] is Ready metadata: creationTimestamp: "2026-07-02T14:30:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:30:09Z" name: isvc-sklearn-graph-raw-hpa-130af.18be7f6d2528efc3 namespace: kserve-ci-e2e-test resourceVersion: "17049" uid: a21cff61-fca4-421d-bec3-dc408114dc69 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:32:23.143877Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6 namespace: kserve-ci-e2e-test resourceVersion: "18294" uid: f90c7b87-d358-4e52-8f9c-f66207d43db1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6 to ip-10-0-132-82.ec2.internal metadata: creationTimestamp: "2026-07-02T14:32:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:32:23Z" name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6.18be7f8c52a36fda namespace: kserve-ci-e2e-test resourceVersion: "18298" uid: 094ca09c-dabc-4f00-9b43-1705158b70f7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75fd45cf98-dpdjc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:32:23Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6 namespace: kserve-ci-e2e-test resourceVersion: "18296" uid: f90c7b87-d358-4e52-8f9c-f66207d43db1 kind: Event lastTimestamp: "2026-07-02T14:32:23Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-scale-raw-a9dfc-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:32:23Z" name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6.18be7f8c5dc02cd6 namespace: kserve-ci-e2e-test resourceVersion: "18324" uid: abcb5620-b9e4-403d-8f2e-311261d2fefa reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:32:24Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6 namespace: kserve-ci-e2e-test resourceVersion: "18309" uid: f90c7b87-d358-4e52-8f9c-f66207d43db1 kind: Event lastTimestamp: "2026-07-02T14:32:24Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:32:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:32:24Z" name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6.18be7f8c8fcb7ead namespace: kserve-ci-e2e-test resourceVersion: "18335" uid: 6a855cd0-b3f1-4f0b-a1a9-c0631853791e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:32:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6 namespace: kserve-ci-e2e-test resourceVersion: "18296" uid: f90c7b87-d358-4e52-8f9c-f66207d43db1 kind: Event lastTimestamp: "2026-07-02T14:32:24Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8e2b89e7009febc315fad3ff8d06e5aec68b35b7cc22df79daf7c1334f909f5a" already present on machine metadata: creationTimestamp: "2026-07-02T14:32:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:32:24Z" name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6.18be7f8c916b2abf namespace: kserve-ci-e2e-test resourceVersion: "18337" uid: 87aa4f0f-19a5-4cb9-a1e7-6f07bd878bf9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:32:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6 namespace: kserve-ci-e2e-test resourceVersion: "18296" uid: f90c7b87-d358-4e52-8f9c-f66207d43db1 kind: Event lastTimestamp: "2026-07-02T14:32:24Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:32:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:32:24Z" name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6.18be7f8c94649d37 namespace: kserve-ci-e2e-test resourceVersion: "18339" uid: 017e76ed-0f73-4ea2-a08e-869aa95ab8e5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:32:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6 namespace: kserve-ci-e2e-test resourceVersion: "18296" uid: f90c7b87-d358-4e52-8f9c-f66207d43db1 kind: Event lastTimestamp: "2026-07-02T14:32:24Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:32:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:32:24Z" name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6.18be7f8c94d1cbd5 namespace: kserve-ci-e2e-test resourceVersion: "18340" uid: 1c9b30a8-b1c8-447a-9148-430c3ae1ffe4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:32:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6 namespace: kserve-ci-e2e-test resourceVersion: "18296" uid: f90c7b87-d358-4e52-8f9c-f66207d43db1 kind: Event lastTimestamp: "2026-07-02T14:32:28Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:32:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:32:28Z" name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6.18be7f8da32b9fad namespace: kserve-ci-e2e-test resourceVersion: "18380" uid: 79809908-01fb-4018-b318-eb953666c677 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:32:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6 namespace: kserve-ci-e2e-test resourceVersion: "18296" uid: f90c7b87-d358-4e52-8f9c-f66207d43db1 kind: Event lastTimestamp: "2026-07-02T14:32:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:32:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:32:28Z" name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6.18be7f8da64b64f0 namespace: kserve-ci-e2e-test resourceVersion: "18385" uid: 270db65b-9ead-4459-ad6a-3eacc748be49 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:32:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6 namespace: kserve-ci-e2e-test resourceVersion: "18296" uid: f90c7b87-d358-4e52-8f9c-f66207d43db1 kind: Event lastTimestamp: "2026-07-02T14:32:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:32:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:32:28Z" name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6.18be7f8da6bf5fb8 namespace: kserve-ci-e2e-test resourceVersion: "18386" uid: b7a89328-d5dc-40d9-87de-14843da5f894 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:32:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6 namespace: kserve-ci-e2e-test resourceVersion: "18296" uid: f90c7b87-d358-4e52-8f9c-f66207d43db1 kind: Event lastTimestamp: "2026-07-02T14:32:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:32:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:32:28Z" name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6.18be7f8da6c9e4a0 namespace: kserve-ci-e2e-test resourceVersion: "18389" uid: 09760b89-9201-4635-b102-d973f834ec83 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:32:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6 namespace: kserve-ci-e2e-test resourceVersion: "18296" uid: f90c7b87-d358-4e52-8f9c-f66207d43db1 kind: Event lastTimestamp: "2026-07-02T14:32:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:32:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:32:28Z" name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6.18be7f8da9fd66b5 namespace: kserve-ci-e2e-test resourceVersion: "18392" uid: 3f8723b1-7027-4787-9d70-d67d1aa6160a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:32:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6 namespace: kserve-ci-e2e-test resourceVersion: "18296" uid: f90c7b87-d358-4e52-8f9c-f66207d43db1 kind: Event lastTimestamp: "2026-07-02T14:32:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:32:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:32:28Z" name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6.18be7f8daa8e730f namespace: kserve-ci-e2e-test resourceVersion: "18393" uid: b3521fd7-14b3-407e-bfb5-8a8ea482ad90 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 18 eventTime: null firstTimestamp: "2026-07-02T14:32:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6 namespace: kserve-ci-e2e-test resourceVersion: "18296" uid: f90c7b87-d358-4e52-8f9c-f66207d43db1 kind: Event lastTimestamp: "2026-07-02T14:35:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:32:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:35:12Z" name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6.18be7f8ddf1e2f6c namespace: kserve-ci-e2e-test resourceVersion: "19624" uid: f84efd76-99d4-4c57-b03f-c0afffe6c74a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:35:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6 namespace: kserve-ci-e2e-test resourceVersion: "18296" uid: f90c7b87-d358-4e52-8f9c-f66207d43db1 kind: Event lastTimestamp: "2026-07-02T14:35:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:35:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:35:03Z" name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6.18be7fb19e513a0e namespace: kserve-ci-e2e-test resourceVersion: "19508" uid: 4a301738-a245-4545-95a1-baf4a41d10d8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:35:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6 namespace: kserve-ci-e2e-test resourceVersion: "18296" uid: f90c7b87-d358-4e52-8f9c-f66207d43db1 kind: Event lastTimestamp: "2026-07-02T14:35:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:35:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:35:03Z" name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6.18be7fb19e519cf3 namespace: kserve-ci-e2e-test resourceVersion: "19509" uid: dc32147e-ee82-4625-93c4-19113a45f23f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:35:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6 namespace: kserve-ci-e2e-test resourceVersion: "18296" uid: f90c7b87-d358-4e52-8f9c-f66207d43db1 kind: Event lastTimestamp: "2026-07-02T14:35:10Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:35:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:35:10Z" name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6.18be7fb231939a19 namespace: kserve-ci-e2e-test resourceVersion: "19612" uid: a6cc483b-b398-4a57-a593-c770d0148e8e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:32:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c namespace: kserve-ci-e2e-test resourceVersion: "18278" uid: c7ff26ef-df0d-4814-9842-0cb6629f94d5 kind: Event lastTimestamp: "2026-07-02T14:32:23Z" message: 'Created pod: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c-2hmp6' metadata: creationTimestamp: "2026-07-02T14:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:32:23Z" name: isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c.18be7f8c523873a4 namespace: kserve-ci-e2e-test resourceVersion: "18295" uid: 21e8bd30-1123-48bb-8571-9a5e62ea66d8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:32:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-a9dfc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18272" uid: 6ee3ef33-b41d-4d6c-814f-b6c62e8d7f2c kind: Event lastTimestamp: "2026-07-02T14:32:23Z" message: Scaled up replica set isvc-sklearn-scale-raw-a9dfc-predictor-7c997cd95c from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:32:23Z" name: isvc-sklearn-scale-raw-a9dfc-predictor.18be7f8c4dfa418a namespace: kserve-ci-e2e-test resourceVersion: "18280" uid: 30f034e6-c2c9-4d2c-969e-44e65f1060b8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:32:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-a9dfc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18276" uid: fc3d880b-691f-4875-89f9-80d945c52f85 kind: Event lastTimestamp: "2026-07-02T14:32:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:32:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:32:38Z" name: isvc-sklearn-scale-raw-a9dfc-predictor.18be7f8fccbfaf5c namespace: kserve-ci-e2e-test resourceVersion: "18466" uid: c105bf3c-8a91-409a-a76e-0606a8e2ce2a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:32:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-a9dfc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18276" uid: fc3d880b-691f-4875-89f9-80d945c52f85 kind: Event lastTimestamp: "2026-07-02T14:32:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:32:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:32:38Z" name: isvc-sklearn-scale-raw-a9dfc-predictor.18be7f8fcd2d40e0 namespace: kserve-ci-e2e-test resourceVersion: "18468" uid: 809b5c05-6033-4e45-91e4-9ff132c598e6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T14:32:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-a9dfc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18467" uid: fc3d880b-691f-4875-89f9-80d945c52f85 kind: Event lastTimestamp: "2026-07-02T14:34:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:32:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:34:53Z" name: isvc-sklearn-scale-raw-a9dfc-predictor.18be7f934c3819c6 namespace: kserve-ci-e2e-test resourceVersion: "19413" uid: c7cddbc8-d8af-4b6e-84e8-111f88903676 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T14:32:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-a9dfc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18467" uid: fc3d880b-691f-4875-89f9-80d945c52f85 kind: Event lastTimestamp: "2026-07-02T14:34:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:32:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:34:53Z" name: isvc-sklearn-scale-raw-a9dfc-predictor.18be7f934caa1ac0 namespace: kserve-ci-e2e-test resourceVersion: "19414" uid: 31f3cdf7-d4ba-45be-b2c0-14710da5c871 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T14:32:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-a9dfc namespace: kserve-ci-e2e-test resourceVersion: "18255" uid: 9aca1dfe-8b7a-403c-98f2-d73b1560fc67 kind: Event lastTimestamp: "2026-07-02T14:35:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:35:02Z" name: isvc-sklearn-scale-raw-a9dfc.18be7f8c4e4eaec4 namespace: kserve-ci-e2e-test resourceVersion: "19484" uid: fafb0cb3-2e77-4010-bf94-5653193447de reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:32:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-a9dfc namespace: kserve-ci-e2e-test resourceVersion: "18255" uid: 9aca1dfe-8b7a-403c-98f2-d73b1560fc67 kind: Event lastTimestamp: "2026-07-02T14:32:23Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-a9dfc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-a9dfc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:32:23Z" name: isvc-sklearn-scale-raw-a9dfc.18be7f8c4ea44dc7 namespace: kserve-ci-e2e-test resourceVersion: "18284" uid: ff8f26e1-dbbc-48ee-a0c8-c66bd7c91370 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:35:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-a9dfc namespace: kserve-ci-e2e-test resourceVersion: "19482" uid: 9aca1dfe-8b7a-403c-98f2-d73b1560fc67 kind: Event lastTimestamp: "2026-07-02T14:35:02Z" message: InferenceService [isvc-sklearn-scale-raw-a9dfc] is Ready metadata: creationTimestamp: "2026-07-02T14:35:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:35:02Z" name: isvc-sklearn-scale-raw-a9dfc.18be7fb15e1af110 namespace: kserve-ci-e2e-test resourceVersion: "19483" uid: 37583b6c-d69e-49d6-bc37-852284efeb72 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:26:57.294895Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq namespace: kserve-ci-e2e-test resourceVersion: "15320" uid: 1d127a37-158a-4a19-afd6-d536738f4108 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq to ip-10-0-138-134.ec2.internal metadata: creationTimestamp: "2026-07-02T14:26:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:26:57Z" name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq.18be7f4074867f18 namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: e452d615-92d2-4d8a-b7d4-f599f8781c05 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75fd45cf98-dpdjc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:57Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq namespace: kserve-ci-e2e-test resourceVersion: "15333" uid: 1d127a37-158a-4a19-afd6-d536738f4108 kind: Event lastTimestamp: "2026-07-02T14:26:57Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:26:57Z" name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq.18be7f408dfedba0 namespace: kserve-ci-e2e-test resourceVersion: "15353" uid: 568d01c8-cf9a-4682-867d-27724925b5e5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 1d127a37-158a-4a19-afd6-d536738f4108 kind: Event lastTimestamp: "2026-07-02T14:26:57Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8e2b89e7009febc315fad3ff8d06e5aec68b35b7cc22df79daf7c1334f909f5a" already present on machine metadata: creationTimestamp: "2026-07-02T14:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:57Z" name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq.18be7f408f706309 namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: 8a778988-d301-479c-9476-b639896f87cf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 1d127a37-158a-4a19-afd6-d536738f4108 kind: Event lastTimestamp: "2026-07-02T14:26:57Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:57Z" name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq.18be7f4092267842 namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: a5db42d7-6fd9-4da7-a21b-5c0034971f83 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 1d127a37-158a-4a19-afd6-d536738f4108 kind: Event lastTimestamp: "2026-07-02T14:26:57Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:57Z" name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq.18be7f409292439b namespace: kserve-ci-e2e-test resourceVersion: "15358" uid: 5b15aeca-722a-4e7d-8d4e-c42d86e59c44 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 1d127a37-158a-4a19-afd6-d536738f4108 kind: Event lastTimestamp: "2026-07-02T14:27:01Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-02T14:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:01Z" name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq.18be7f41628fe8af namespace: kserve-ci-e2e-test resourceVersion: "15394" uid: fad58683-f3f6-4606-994f-087655178b33 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 1d127a37-158a-4a19-afd6-d536738f4108 kind: Event lastTimestamp: "2026-07-02T14:27:24Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 22.775s (22.775s including waiting). Image size: 1305293048 bytes.' metadata: creationTimestamp: "2026-07-02T14:27:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:24Z" name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq.18be7f46b01c9c0f namespace: kserve-ci-e2e-test resourceVersion: "15580" uid: d42deb03-39dc-46e3-870b-e8c4765d1cd8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 1d127a37-158a-4a19-afd6-d536738f4108 kind: Event lastTimestamp: "2026-07-02T14:27:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:27:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:24Z" name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq.18be7f46b469e709 namespace: kserve-ci-e2e-test resourceVersion: "15581" uid: 29df03f6-1a5d-496e-a15d-b3479c73d4ee reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 1d127a37-158a-4a19-afd6-d536738f4108 kind: Event lastTimestamp: "2026-07-02T14:27:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:27:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:24Z" name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq.18be7f46b4db7c61 namespace: kserve-ci-e2e-test resourceVersion: "15582" uid: cb195f50-ea3e-42e0-8c3b-bb7f618ef119 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 1d127a37-158a-4a19-afd6-d536738f4108 kind: Event lastTimestamp: "2026-07-02T14:27:24Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T14:27:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:24Z" name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq.18be7f46b4e8e8a9 namespace: kserve-ci-e2e-test resourceVersion: "15583" uid: 5fa610d7-a2bc-4cab-b118-0e5a3311b0c0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 1d127a37-158a-4a19-afd6-d536738f4108 kind: Event lastTimestamp: "2026-07-02T14:27:26Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.362s (2.362s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T14:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:26Z" name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq.18be7f4741c11d2b namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: 58a6dc82-5176-4d54-8c52-50085b74e088 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 1d127a37-158a-4a19-afd6-d536738f4108 kind: Event lastTimestamp: "2026-07-02T14:27:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:26Z" name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq.18be7f4744a839cd namespace: kserve-ci-e2e-test resourceVersion: "15600" uid: 059fe256-96e3-4f47-9ce6-35402daf070d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 1d127a37-158a-4a19-afd6-d536738f4108 kind: Event lastTimestamp: "2026-07-02T14:27:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:26Z" name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq.18be7f474517d841 namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 92b92cd7-3749-4ebc-af15-c8a2f3808405 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T14:27:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 1d127a37-158a-4a19-afd6-d536738f4108 kind: Event lastTimestamp: "2026-07-02T14:28:14Z" message: 'Readiness probe failed: dial tcp 10.133.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:27:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:14Z" name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq.18be7f47b2883e0b namespace: kserve-ci-e2e-test resourceVersion: "15963" uid: c77a2577-257b-4f43-ae11-f1ea6988c596 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 1d127a37-158a-4a19-afd6-d536738f4108 kind: Event lastTimestamp: "2026-07-02T14:28:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:47Z" name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq.18be7f5a25b5241b namespace: kserve-ci-e2e-test resourceVersion: "16360" uid: 2c80bf8b-7dc9-48a1-ab27-884085c84920 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 1d127a37-158a-4a19-afd6-d536738f4108 kind: Event lastTimestamp: "2026-07-02T14:28:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:47Z" name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq.18be7f5a25b576f8 namespace: kserve-ci-e2e-test resourceVersion: "16362" uid: fc675c21-d979-4f69-8a9e-4c76f2fd3322 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 1d127a37-158a-4a19-afd6-d536738f4108 kind: Event lastTimestamp: "2026-07-02T14:28:49Z" message: 'Readiness probe failed: Get "https://10.133.0.22:8643/healthz": dial tcp 10.133.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:28:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:49Z" name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq.18be7f5a8ebb08eb namespace: kserve-ci-e2e-test resourceVersion: "16412" uid: 09ce2c45-da70-493f-a7f5-11e17918c9dd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7 namespace: kserve-ci-e2e-test resourceVersion: "15312" uid: 76a9c22d-7d2b-4475-bb14-9c37946106ea kind: Event lastTimestamp: "2026-07-02T14:26:57Z" message: 'Created pod: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7-xb6jq' metadata: creationTimestamp: "2026-07-02T14:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:26:57Z" name: isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7.18be7f40741e88e4 namespace: kserve-ci-e2e-test resourceVersion: "15322" uid: d53d4625-9dea-47e2-91e1-bc28cd810f93 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-042a3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15309" uid: 56f929ab-6ddb-478c-bff4-0d913ed74799 kind: Event lastTimestamp: "2026-07-02T14:26:57Z" message: Scaled up replica set isvc-xgboost-graph-raw-042a3-predictor-77d54fc7f7 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:26:57Z" name: isvc-xgboost-graph-raw-042a3-predictor.18be7f407179df11 namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: 35ec3944-139e-402a-ba45-1342c67f9ed3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:27:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-042a3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15311" uid: 088fe115-cc2c-4bbd-af7f-b548fbb82d66 kind: Event lastTimestamp: "2026-07-02T14:27:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:57Z" name: isvc-xgboost-graph-raw-042a3-predictor.18be7f43eff4efc2 namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 2cfdbe96-b3c7-48d9-864c-a90999889d1e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:27:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-042a3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15311" uid: 088fe115-cc2c-4bbd-af7f-b548fbb82d66 kind: Event lastTimestamp: "2026-07-02T14:27:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:57Z" name: isvc-xgboost-graph-raw-042a3-predictor.18be7f43f05a8581 namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: 5da49161-d78a-459c-8cf9-3d699f5af115 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:28:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-042a3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15494" uid: 088fe115-cc2c-4bbd-af7f-b548fbb82d66 kind: Event lastTimestamp: "2026-07-02T14:28:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:28:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:42Z" name: isvc-xgboost-graph-raw-042a3-predictor.18be7f51eb0e124d namespace: kserve-ci-e2e-test resourceVersion: "16216" uid: c97ea12b-7683-4118-9260-41d00d1368df reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:28:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-042a3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15494" uid: 088fe115-cc2c-4bbd-af7f-b548fbb82d66 kind: Event lastTimestamp: "2026-07-02T14:28:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:28:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:42Z" name: isvc-xgboost-graph-raw-042a3-predictor.18be7f51eb7ede86 namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: f1630cb5-2d62-44e2-afc1-6fa3f8721a08 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T14:26:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-042a3 namespace: kserve-ci-e2e-test resourceVersion: "15291" uid: 14ebfdfb-167e-49b7-a1ed-d93d4c9d21ec kind: Event lastTimestamp: "2026-07-02T14:28:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:28:24Z" name: isvc-xgboost-graph-raw-042a3.18be7f40717e3b23 namespace: kserve-ci-e2e-test resourceVersion: "16051" uid: 901f749d-4e4f-49d1-9954-e338495c8e1b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-042a3 namespace: kserve-ci-e2e-test resourceVersion: "16048" uid: 14ebfdfb-167e-49b7-a1ed-d93d4c9d21ec kind: Event lastTimestamp: "2026-07-02T14:28:24Z" message: InferenceService [isvc-xgboost-graph-raw-042a3] is Ready metadata: creationTimestamp: "2026-07-02T14:28:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:28:24Z" name: isvc-xgboost-graph-raw-042a3.18be7f54c1bf4129 namespace: kserve-ci-e2e-test resourceVersion: "16049" uid: 6787e999-08e1-43e7-bb35-e9430e62c717 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:28:47.608487Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: 04b57e6f-ffe2-4605-b2b0-365569b1bfd2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg to ip-10-0-138-134.ec2.internal metadata: creationTimestamp: "2026-07-02T14:28:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:28:47Z" name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg.18be7f5a23ba5d4b namespace: kserve-ci-e2e-test resourceVersion: "16347" uid: 103b61de-3fc2-4307-bfa3-b1c5ea2ab518 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75fd45cf98-dpdjc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:48Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg namespace: kserve-ci-e2e-test resourceVersion: "16359" uid: 04b57e6f-ffe2-4605-b2b0-365569b1bfd2 kind: Event lastTimestamp: "2026-07-02T14:28:48Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:28:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:28:48Z" name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg.18be7f5a3d194603 namespace: kserve-ci-e2e-test resourceVersion: "16378" uid: fea46376-7612-46ee-8ae7-db658108dca1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: 04b57e6f-ffe2-4605-b2b0-365569b1bfd2 kind: Event lastTimestamp: "2026-07-02T14:28:48Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8e2b89e7009febc315fad3ff8d06e5aec68b35b7cc22df79daf7c1334f909f5a" already present on machine metadata: creationTimestamp: "2026-07-02T14:28:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:48Z" name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg.18be7f5a3ea1004d namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: b69ced07-f5ce-411c-9038-16e307cb7a41 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: 04b57e6f-ffe2-4605-b2b0-365569b1bfd2 kind: Event lastTimestamp: "2026-07-02T14:28:48Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:28:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:48Z" name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg.18be7f5a415632a4 namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: 2177de1a-ebb3-46c5-8c00-5dd11f4ebaed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: 04b57e6f-ffe2-4605-b2b0-365569b1bfd2 kind: Event lastTimestamp: "2026-07-02T14:28:48Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:28:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:48Z" name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg.18be7f5a41b81bdc namespace: kserve-ci-e2e-test resourceVersion: "16382" uid: 98285caf-e90a-49a0-9073-34cdad0876dd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: 04b57e6f-ffe2-4605-b2b0-365569b1bfd2 kind: Event lastTimestamp: "2026-07-02T14:28:52Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-07-02T14:28:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:52Z" name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg.18be7f5b54bc68df namespace: kserve-ci-e2e-test resourceVersion: "16441" uid: 0eead1ed-563e-49ec-b972-38f00522ae73 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: 04b57e6f-ffe2-4605-b2b0-365569b1bfd2 kind: Event lastTimestamp: "2026-07-02T14:28:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:28:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:52Z" name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg.18be7f5b57338996 namespace: kserve-ci-e2e-test resourceVersion: "16443" uid: cd6cfaff-5391-4edb-a83d-923a729370e1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: 04b57e6f-ffe2-4605-b2b0-365569b1bfd2 kind: Event lastTimestamp: "2026-07-02T14:28:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:28:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:52Z" name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg.18be7f5b5794abbc namespace: kserve-ci-e2e-test resourceVersion: "16444" uid: 524945bd-712d-4ad7-8206-c51689a10bd6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: 04b57e6f-ffe2-4605-b2b0-365569b1bfd2 kind: Event lastTimestamp: "2026-07-02T14:28:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:28:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:52Z" name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg.18be7f5b579f7960 namespace: kserve-ci-e2e-test resourceVersion: "16445" uid: dae1592c-0f49-418c-a023-9e4115734163 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: 04b57e6f-ffe2-4605-b2b0-365569b1bfd2 kind: Event lastTimestamp: "2026-07-02T14:28:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:28:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:52Z" name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg.18be7f5b5b28a3e8 namespace: kserve-ci-e2e-test resourceVersion: "16450" uid: c73d432d-e594-40d1-8153-4cc298553828 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: 04b57e6f-ffe2-4605-b2b0-365569b1bfd2 kind: Event lastTimestamp: "2026-07-02T14:28:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:28:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:52Z" name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg.18be7f5b5b8d6a01 namespace: kserve-ci-e2e-test resourceVersion: "16451" uid: ae3d6d3d-ba17-470d-b41c-2f9faa16d2cc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T14:28:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: 04b57e6f-ffe2-4605-b2b0-365569b1bfd2 kind: Event lastTimestamp: "2026-07-02T14:30:40Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:28:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:40Z" name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg.18be7f5bcc7f8771 namespace: kserve-ci-e2e-test resourceVersion: "17418" uid: 8b0a5ca4-983f-486e-880b-f374f7fb7641 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: 04b57e6f-ffe2-4605-b2b0-365569b1bfd2 kind: Event lastTimestamp: "2026-07-02T14:30:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:30:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:37Z" name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg.18be7f73d3457839 namespace: kserve-ci-e2e-test resourceVersion: "17373" uid: 457b6817-0832-44ed-9e58-0c321358073d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: 04b57e6f-ffe2-4605-b2b0-365569b1bfd2 kind: Event lastTimestamp: "2026-07-02T14:30:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:30:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:37Z" name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg.18be7f73d34604e6 namespace: kserve-ci-e2e-test resourceVersion: "17374" uid: 3f890ad9-9cd1-4325-901b-505ab1b4880f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: 04b57e6f-ffe2-4605-b2b0-365569b1bfd2 kind: Event lastTimestamp: "2026-07-02T14:30:40Z" message: 'Readiness probe failed: Get "https://10.133.0.23:8643/healthz": dial tcp 10.133.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:30:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:40Z" name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg.18be7f747ac78d5d namespace: kserve-ci-e2e-test resourceVersion: "17417" uid: e2ca08be-5aa9-42ef-ace7-cac7944d02ea reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-134.ec2.internal source: component: kubelet host: ip-10-0-138-134.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf namespace: kserve-ci-e2e-test resourceVersion: "16324" uid: 8a056ceb-605a-4faa-8cf9-4b4e0a33e160 kind: Event lastTimestamp: "2026-07-02T14:28:47Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf-l7dvg' metadata: creationTimestamp: "2026-07-02T14:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:47Z" name: isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf.18be7f5a233fd6e3 namespace: kserve-ci-e2e-test resourceVersion: "16340" uid: 0aafa09a-723a-4f6b-9338-9d8a79431618 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-130af-predictor namespace: kserve-ci-e2e-test resourceVersion: "16319" uid: 8bd64c2c-23ed-4cd3-8fc5-15a62a23db0c kind: Event lastTimestamp: "2026-07-02T14:28:47Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-130af-predictor-86bb7cf7bf from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:47Z" name: isvc-xgboost-graph-raw-hpa-130af-predictor.18be7f5a20822437 namespace: kserve-ci-e2e-test resourceVersion: "16325" uid: 96a5f9bf-6141-4263-9702-9e8086d200c3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:29:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-130af-predictor namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: 80a9108b-0cab-4220-823b-8db36c1c038f kind: Event lastTimestamp: "2026-07-02T14:29:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:29:17Z" name: isvc-xgboost-graph-raw-hpa-130af-predictor.18be7f5d9eecbe38 namespace: kserve-ci-e2e-test resourceVersion: "16659" uid: 0b5cab24-61c1-4545-bb4a-4d742206bb99 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:29:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-130af-predictor namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: 80a9108b-0cab-4220-823b-8db36c1c038f kind: Event lastTimestamp: "2026-07-02T14:29:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:29:17Z" name: isvc-xgboost-graph-raw-hpa-130af-predictor.18be7f5d9f58b7e9 namespace: kserve-ci-e2e-test resourceVersion: "16660" uid: 28a96ebc-8b4c-456d-a297-c0bace4e0f93 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:29:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-130af-predictor namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: 80a9108b-0cab-4220-823b-8db36c1c038f kind: Event lastTimestamp: "2026-07-02T14:30:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:29:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:30:17Z" name: isvc-xgboost-graph-raw-hpa-130af-predictor.18be7f649cb0c71f namespace: kserve-ci-e2e-test resourceVersion: "17108" uid: 6b78a332-95f3-4f36-950e-b2823d069d87 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:29:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-130af-predictor namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: 80a9108b-0cab-4220-823b-8db36c1c038f kind: Event lastTimestamp: "2026-07-02T14:30:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:29:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:30:17Z" name: isvc-xgboost-graph-raw-hpa-130af-predictor.18be7f649d18e1f8 namespace: kserve-ci-e2e-test resourceVersion: "17109" uid: 6537412c-5bc2-4d4b-a3f7-1e22ef995b05 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T14:28:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-130af namespace: kserve-ci-e2e-test resourceVersion: "16287" uid: 020ab554-525f-48c3-9217-9e20f0a3c76e kind: Event lastTimestamp: "2026-07-02T14:30:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:30:00Z" name: isvc-xgboost-graph-raw-hpa-130af.18be7f5a207983da namespace: kserve-ci-e2e-test resourceVersion: "16981" uid: 448dab3d-fae4-4443-839b-12f4059400ac reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-130af namespace: kserve-ci-e2e-test resourceVersion: "16287" uid: 020ab554-525f-48c3-9217-9e20f0a3c76e kind: Event lastTimestamp: "2026-07-02T14:28:47Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-hpa-130af": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-hpa-130af": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:28:47Z" name: isvc-xgboost-graph-raw-hpa-130af.18be7f5a20f6b52f namespace: kserve-ci-e2e-test resourceVersion: "16328" uid: d66052e4-9538-495e-9bf3-8fc41b569065 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-130af namespace: kserve-ci-e2e-test resourceVersion: "16979" uid: 020ab554-525f-48c3-9217-9e20f0a3c76e kind: Event lastTimestamp: "2026-07-02T14:30:00Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-130af] is Ready metadata: creationTimestamp: "2026-07-02T14:30:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:30:00Z" name: isvc-xgboost-graph-raw-hpa-130af.18be7f6b30466c52 namespace: kserve-ci-e2e-test resourceVersion: "16980" uid: d22a1b06-a705-4b08-a453-57065d61293d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:30:37.794507Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-f89aa-predictor-7b4979f9b7-75gvg namespace: kserve-ci-e2e-test resourceVersion: "17327" uid: d24544ce-6fc5-4cd7-a1fb-1f7fc65702de kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-f89aa-predictor-7b4979f9b7-75gvg to ip-10-0-132-82.ec2.internal metadata: creationTimestamp: "2026-07-02T14:30:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:30:37Z" name: message-dumper-raw-f89aa-predictor-7b4979f9b7-75gvg.18be7f73cb538d02 namespace: kserve-ci-e2e-test resourceVersion: "17331" uid: 2eedf4cf-40fd-45cf-8dc3-b2b3e94d2312 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75fd45cf98-dpdjc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:37Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-f89aa-predictor-7b4979f9b7-75gvg namespace: kserve-ci-e2e-test resourceVersion: "17328" uid: d24544ce-6fc5-4cd7-a1fb-1f7fc65702de kind: Event lastTimestamp: "2026-07-02T14:30:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "message-dumper-raw-f89aa-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:30:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:37Z" name: message-dumper-raw-f89aa-predictor-7b4979f9b7-75gvg.18be7f73d3e37e5b namespace: kserve-ci-e2e-test resourceVersion: "17375" uid: 417edec0-7390-414a-aedb-0971c77ad9c4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:38Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-f89aa-predictor-7b4979f9b7-75gvg namespace: kserve-ci-e2e-test resourceVersion: "17337" uid: d24544ce-6fc5-4cd7-a1fb-1f7fc65702de kind: Event lastTimestamp: "2026-07-02T14:30:38Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:30:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:30:38Z" name: message-dumper-raw-f89aa-predictor-7b4979f9b7-75gvg.18be7f740993d2f3 namespace: kserve-ci-e2e-test resourceVersion: "17390" uid: 515b9968-684f-42e9-8544-469b55db1d4e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-f89aa-predictor-7b4979f9b7-75gvg namespace: kserve-ci-e2e-test resourceVersion: "17328" uid: d24544ce-6fc5-4cd7-a1fb-1f7fc65702de kind: Event lastTimestamp: "2026-07-02T14:30:38Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-02T14:30:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:38Z" name: message-dumper-raw-f89aa-predictor-7b4979f9b7-75gvg.18be7f740af71d8e namespace: kserve-ci-e2e-test resourceVersion: "17392" uid: d930eb97-914e-4184-b838-047e1e0eed03 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-f89aa-predictor-7b4979f9b7-75gvg namespace: kserve-ci-e2e-test resourceVersion: "17328" uid: d24544ce-6fc5-4cd7-a1fb-1f7fc65702de kind: Event lastTimestamp: "2026-07-02T14:30:40Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.266s (1.266s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-02T14:30:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:40Z" name: message-dumper-raw-f89aa-predictor-7b4979f9b7-75gvg.18be7f745670c91e namespace: kserve-ci-e2e-test resourceVersion: "17402" uid: 5948f2c1-21c3-4b36-9528-9858f3aa4418 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-f89aa-predictor-7b4979f9b7-75gvg namespace: kserve-ci-e2e-test resourceVersion: "17328" uid: d24544ce-6fc5-4cd7-a1fb-1f7fc65702de kind: Event lastTimestamp: "2026-07-02T14:30:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:30:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:40Z" name: message-dumper-raw-f89aa-predictor-7b4979f9b7-75gvg.18be7f745a8e452c namespace: kserve-ci-e2e-test resourceVersion: "17404" uid: a7e97b69-30c0-433e-9903-b21a2ed63564 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-f89aa-predictor-7b4979f9b7-75gvg namespace: kserve-ci-e2e-test resourceVersion: "17328" uid: d24544ce-6fc5-4cd7-a1fb-1f7fc65702de kind: Event lastTimestamp: "2026-07-02T14:30:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:30:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:40Z" name: message-dumper-raw-f89aa-predictor-7b4979f9b7-75gvg.18be7f745afc91f5 namespace: kserve-ci-e2e-test resourceVersion: "17405" uid: a28674f9-e182-4538-86b5-81f2f9f48d52 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-f89aa-predictor-7b4979f9b7-75gvg namespace: kserve-ci-e2e-test resourceVersion: "17328" uid: d24544ce-6fc5-4cd7-a1fb-1f7fc65702de kind: Event lastTimestamp: "2026-07-02T14:30:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:30:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:40Z" name: message-dumper-raw-f89aa-predictor-7b4979f9b7-75gvg.18be7f745b05c0ba namespace: kserve-ci-e2e-test resourceVersion: "17406" uid: 1b90508f-aec7-4a95-832e-43f41a6687cf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-f89aa-predictor-7b4979f9b7-75gvg namespace: kserve-ci-e2e-test resourceVersion: "17328" uid: d24544ce-6fc5-4cd7-a1fb-1f7fc65702de kind: Event lastTimestamp: "2026-07-02T14:30:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:30:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:40Z" name: message-dumper-raw-f89aa-predictor-7b4979f9b7-75gvg.18be7f745da81f9c namespace: kserve-ci-e2e-test resourceVersion: "17407" uid: cfe53098-c849-40b9-bb38-5e99dfe4448d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-f89aa-predictor-7b4979f9b7-75gvg namespace: kserve-ci-e2e-test resourceVersion: "17328" uid: d24544ce-6fc5-4cd7-a1fb-1f7fc65702de kind: Event lastTimestamp: "2026-07-02T14:30:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:30:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:40Z" name: message-dumper-raw-f89aa-predictor-7b4979f9b7-75gvg.18be7f745e19312d namespace: kserve-ci-e2e-test resourceVersion: "17408" uid: 5db1518e-f19e-4de7-8f6f-734ced5879a0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-f89aa-predictor-7b4979f9b7-75gvg namespace: kserve-ci-e2e-test resourceVersion: "17328" uid: d24544ce-6fc5-4cd7-a1fb-1f7fc65702de kind: Event lastTimestamp: "2026-07-02T14:32:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:32:23Z" name: message-dumper-raw-f89aa-predictor-7b4979f9b7-75gvg.18be7f8c56a63374 namespace: kserve-ci-e2e-test resourceVersion: "18319" uid: 806560b5-2503-4a77-9a9c-6758dbebcd11 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:32:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-f89aa-predictor-7b4979f9b7-75gvg namespace: kserve-ci-e2e-test resourceVersion: "17328" uid: d24544ce-6fc5-4cd7-a1fb-1f7fc65702de kind: Event lastTimestamp: "2026-07-02T14:32:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:32:23Z" name: message-dumper-raw-f89aa-predictor-7b4979f9b7-75gvg.18be7f8c56a6a8bf namespace: kserve-ci-e2e-test resourceVersion: "18320" uid: 90a70298-1daa-44e6-adee-c8b135a954bd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-f89aa-predictor-7b4979f9b7 namespace: kserve-ci-e2e-test resourceVersion: "17317" uid: 5deb8a1b-a228-4ec4-bfbd-d48460a6e0c1 kind: Event lastTimestamp: "2026-07-02T14:30:37Z" message: 'Created pod: message-dumper-raw-f89aa-predictor-7b4979f9b7-75gvg' metadata: creationTimestamp: "2026-07-02T14:30:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:30:37Z" name: message-dumper-raw-f89aa-predictor-7b4979f9b7.18be7f73cafb454a namespace: kserve-ci-e2e-test resourceVersion: "17329" uid: b8ed668a-9a4f-4626-80f6-e8e22e9df465 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-f89aa-predictor namespace: kserve-ci-e2e-test resourceVersion: "17315" uid: 8988735e-2841-42d3-9706-99cbfef79c3a kind: Event lastTimestamp: "2026-07-02T14:30:37Z" message: Scaled up replica set message-dumper-raw-f89aa-predictor-7b4979f9b7 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:30:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:30:37Z" name: message-dumper-raw-f89aa-predictor.18be7f73c744d94e namespace: kserve-ci-e2e-test resourceVersion: "17318" uid: 99da086f-67b9-4ff3-8c06-92b8ed5ed393 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:30:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-f89aa-predictor namespace: kserve-ci-e2e-test resourceVersion: "17316" uid: 034a2269-3078-4a0d-a486-bf05b68d93a4 kind: Event lastTimestamp: "2026-07-02T14:31:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:30:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:31:07Z" name: message-dumper-raw-f89aa-predictor.18be7f77460a5adf namespace: kserve-ci-e2e-test resourceVersion: "17701" uid: 4fa5ec4f-1536-4515-885a-402f6312d163 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:30:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-f89aa-predictor namespace: kserve-ci-e2e-test resourceVersion: "17316" uid: 034a2269-3078-4a0d-a486-bf05b68d93a4 kind: Event lastTimestamp: "2026-07-02T14:31:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:30:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:31:07Z" name: message-dumper-raw-f89aa-predictor.18be7f77466f39ce namespace: kserve-ci-e2e-test resourceVersion: "17703" uid: fde43ee9-8148-4a39-8048-025e205dad24 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-02T14:30:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-f89aa namespace: kserve-ci-e2e-test resourceVersion: "17308" uid: e9bd52be-6341-4f79-bc72-f68ef2a59c6b kind: Event lastTimestamp: "2026-07-02T14:30:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:30:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:30:47Z" name: message-dumper-raw-f89aa.18be7f73c74640f4 namespace: kserve-ci-e2e-test resourceVersion: "17483" uid: 284089dd-73fd-4abb-b7d0-22c2f7520b23 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-f89aa namespace: kserve-ci-e2e-test resourceVersion: "17308" uid: e9bd52be-6341-4f79-bc72-f68ef2a59c6b kind: Event lastTimestamp: "2026-07-02T14:30:37Z" message: 'Failed to update status for InferenceService "message-dumper-raw-f89aa": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-f89aa": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:30:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:30:37Z" name: message-dumper-raw-f89aa.18be7f73c79f921a namespace: kserve-ci-e2e-test resourceVersion: "17321" uid: a5a486d8-9735-45a8-9135-7bc5210a0bae reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-f89aa namespace: kserve-ci-e2e-test resourceVersion: "17481" uid: e9bd52be-6341-4f79-bc72-f68ef2a59c6b kind: Event lastTimestamp: "2026-07-02T14:30:47Z" message: InferenceService [message-dumper-raw-f89aa] is Ready metadata: creationTimestamp: "2026-07-02T14:30:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:30:47Z" name: message-dumper-raw-f89aa.18be7f760c7befa0 namespace: kserve-ci-e2e-test resourceVersion: "17482" uid: 09898a4c-d384-4f5c-85b1-c28c81a59435 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:28:37.225889Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-042a3-69d995f957-zhmlh namespace: kserve-ci-e2e-test resourceVersion: "16152" uid: d5c8b9ee-6402-4969-9348-68c8986206f9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-042a3-69d995f957-zhmlh to ip-10-0-136-70.ec2.internal metadata: creationTimestamp: "2026-07-02T14:28:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:28:37Z" name: model-chainer-raw-042a3-69d995f957-zhmlh.18be7f57b8e077c1 namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 8a56a7ce-d33a-4091-87f9-b3fbc7a6497e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75fd45cf98-dpdjc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:37Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-042a3-69d995f957-zhmlh namespace: kserve-ci-e2e-test resourceVersion: "16155" uid: d5c8b9ee-6402-4969-9348-68c8986206f9 kind: Event lastTimestamp: "2026-07-02T14:28:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-042a3-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:28:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:37Z" name: model-chainer-raw-042a3-69d995f957-zhmlh.18be7f57c4294596 namespace: kserve-ci-e2e-test resourceVersion: "16167" uid: 5c088754-88c5-4fca-b008-900cff6364f2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-70.ec2.internal source: component: kubelet host: ip-10-0-136-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:38Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-042a3-69d995f957-zhmlh namespace: kserve-ci-e2e-test resourceVersion: "16161" uid: d5c8b9ee-6402-4969-9348-68c8986206f9 kind: Event lastTimestamp: "2026-07-02T14:28:38Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:28:38Z" name: model-chainer-raw-042a3-69d995f957-zhmlh.18be7f57f6a3a6b7 namespace: kserve-ci-e2e-test resourceVersion: "16177" uid: 7df159a6-5168-43c4-9489-6906f975bd20 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-042a3} kind: Pod name: model-chainer-raw-042a3-69d995f957-zhmlh namespace: kserve-ci-e2e-test resourceVersion: "16155" uid: d5c8b9ee-6402-4969-9348-68c8986206f9 kind: Event lastTimestamp: "2026-07-02T14:28:38Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:e0bdb71677d5eb7ee4784378e591e8126d0dcafbbe991e5dcf178c67e85889cd" metadata: creationTimestamp: "2026-07-02T14:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:38Z" name: model-chainer-raw-042a3-69d995f957-zhmlh.18be7f57f8033c9a namespace: kserve-ci-e2e-test resourceVersion: "16179" uid: ddfa1502-f910-432a-9e40-5aef83ce695f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-70.ec2.internal source: component: kubelet host: ip-10-0-136-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-042a3} kind: Pod name: model-chainer-raw-042a3-69d995f957-zhmlh namespace: kserve-ci-e2e-test resourceVersion: "16155" uid: d5c8b9ee-6402-4969-9348-68c8986206f9 kind: Event lastTimestamp: "2026-07-02T14:28:40Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:e0bdb71677d5eb7ee4784378e591e8126d0dcafbbe991e5dcf178c67e85889cd" in 2.564s (2.564s including waiting). Image size: 218068126 bytes.' metadata: creationTimestamp: "2026-07-02T14:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:40Z" name: model-chainer-raw-042a3-69d995f957-zhmlh.18be7f5890da1926 namespace: kserve-ci-e2e-test resourceVersion: "16195" uid: 0e993175-2102-4ff3-aa21-e7b5870d7e23 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-70.ec2.internal source: component: kubelet host: ip-10-0-136-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-042a3} kind: Pod name: model-chainer-raw-042a3-69d995f957-zhmlh namespace: kserve-ci-e2e-test resourceVersion: "16155" uid: d5c8b9ee-6402-4969-9348-68c8986206f9 kind: Event lastTimestamp: "2026-07-02T14:28:40Z" message: 'Created container: model-chainer-raw-042a3' metadata: creationTimestamp: "2026-07-02T14:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:40Z" name: model-chainer-raw-042a3-69d995f957-zhmlh.18be7f58949d66b4 namespace: kserve-ci-e2e-test resourceVersion: "16196" uid: bc7bcd48-0b16-46e4-92de-6e7bc40f5571 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-70.ec2.internal source: component: kubelet host: ip-10-0-136-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-042a3} kind: Pod name: model-chainer-raw-042a3-69d995f957-zhmlh namespace: kserve-ci-e2e-test resourceVersion: "16155" uid: d5c8b9ee-6402-4969-9348-68c8986206f9 kind: Event lastTimestamp: "2026-07-02T14:28:40Z" message: Started container model-chainer-raw-042a3 metadata: creationTimestamp: "2026-07-02T14:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:40Z" name: model-chainer-raw-042a3-69d995f957-zhmlh.18be7f589514164d namespace: kserve-ci-e2e-test resourceVersion: "16197" uid: 7de0128e-06a9-43d6-93fb-5e94a9b3e403 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-70.ec2.internal source: component: kubelet host: ip-10-0-136-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-042a3} kind: Pod name: model-chainer-raw-042a3-69d995f957-zhmlh namespace: kserve-ci-e2e-test resourceVersion: "16155" uid: d5c8b9ee-6402-4969-9348-68c8986206f9 kind: Event lastTimestamp: "2026-07-02T14:28:47Z" message: Stopping container model-chainer-raw-042a3 metadata: creationTimestamp: "2026-07-02T14:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:47Z" name: model-chainer-raw-042a3-69d995f957-zhmlh.18be7f5a113dc7db namespace: kserve-ci-e2e-test resourceVersion: "16263" uid: 16854936-3261-41b8-816f-5d9a46351727 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-70.ec2.internal source: component: kubelet host: ip-10-0-136-70.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T14:28:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-042a3} kind: Pod name: model-chainer-raw-042a3-69d995f957-zhmlh namespace: kserve-ci-e2e-test resourceVersion: "16155" uid: d5c8b9ee-6402-4969-9348-68c8986206f9 kind: Event lastTimestamp: "2026-07-02T14:29:17Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:29:17Z" name: model-chainer-raw-042a3-69d995f957-zhmlh.18be7f5a11944099 namespace: kserve-ci-e2e-test resourceVersion: "16656" uid: 6d5f0cb8-cb88-4378-9e22-11c5e39fabe5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-70.ec2.internal source: component: kubelet host: ip-10-0-136-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-042a3-69d995f957 namespace: kserve-ci-e2e-test resourceVersion: "16148" uid: ab6cbf8e-bfad-40cf-96ba-d9e7de526564 kind: Event lastTimestamp: "2026-07-02T14:28:37Z" message: 'Created pod: model-chainer-raw-042a3-69d995f957-zhmlh' metadata: creationTimestamp: "2026-07-02T14:28:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:37Z" name: model-chainer-raw-042a3-69d995f957.18be7f57b852cb46 namespace: kserve-ci-e2e-test resourceVersion: "16154" uid: 887a046c-d3f2-4b9e-97ce-9b78b7fdf5d6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-042a3 namespace: kserve-ci-e2e-test resourceVersion: "16146" uid: e7b61f6e-de42-43b8-8512-d3389e14c371 kind: Event lastTimestamp: "2026-07-02T14:28:37Z" message: Scaled up replica set model-chainer-raw-042a3-69d995f957 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:28:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:37Z" name: model-chainer-raw-042a3.18be7f57b764c129 namespace: kserve-ci-e2e-test resourceVersion: "16150" uid: 3f28fa85-717a-4394-990f-891d023332fb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-042a3 namespace: kserve-ci-e2e-test resourceVersion: "16163" uid: 8a0ab728-893f-40da-8eb8-16720dd1e704 kind: Event lastTimestamp: "2026-07-02T14:28:37Z" message: InferenceGraph [model-chainer-raw-042a3] is Ready metadata: creationTimestamp: "2026-07-02T14:28:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:28:37Z" name: model-chainer-raw-042a3.18be7f57bd13cbb5 namespace: kserve-ci-e2e-test resourceVersion: "16165" uid: a5b672b7-39cf-48df-8ece-7d4b42bef653 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:30:27.540755Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-130af-6c866bc956-szxj6 namespace: kserve-ci-e2e-test resourceVersion: "17189" uid: b94d640e-9e9d-4c49-adca-66c3a9934370 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-130af-6c866bc956-szxj6 to ip-10-0-136-70.ec2.internal metadata: creationTimestamp: "2026-07-02T14:30:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:30:27Z" name: model-chainer-raw-hpa-130af-6c866bc956-szxj6.18be7f716827d008 namespace: kserve-ci-e2e-test resourceVersion: "17194" uid: 98c439ca-3787-4181-bf8f-f420c4fab58e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75fd45cf98-dpdjc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:28Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-130af-6c866bc956-szxj6 namespace: kserve-ci-e2e-test resourceVersion: "17198" uid: b94d640e-9e9d-4c49-adca-66c3a9934370 kind: Event lastTimestamp: "2026-07-02T14:30:28Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:30:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:30:28Z" name: model-chainer-raw-hpa-130af-6c866bc956-szxj6.18be7f7183ebb64f namespace: kserve-ci-e2e-test resourceVersion: "17210" uid: b7bc0f1a-753b-44e0-a200-40aa8d46d0f9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-130af} kind: Pod name: model-chainer-raw-hpa-130af-6c866bc956-szxj6 namespace: kserve-ci-e2e-test resourceVersion: "17192" uid: b94d640e-9e9d-4c49-adca-66c3a9934370 kind: Event lastTimestamp: "2026-07-02T14:30:28Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:e0bdb71677d5eb7ee4784378e591e8126d0dcafbbe991e5dcf178c67e85889cd" already present on machine metadata: creationTimestamp: "2026-07-02T14:30:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:28Z" name: model-chainer-raw-hpa-130af-6c866bc956-szxj6.18be7f7185433529 namespace: kserve-ci-e2e-test resourceVersion: "17212" uid: 4d45db77-71d0-4988-807e-78c989cc42bd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-70.ec2.internal source: component: kubelet host: ip-10-0-136-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-130af} kind: Pod name: model-chainer-raw-hpa-130af-6c866bc956-szxj6 namespace: kserve-ci-e2e-test resourceVersion: "17192" uid: b94d640e-9e9d-4c49-adca-66c3a9934370 kind: Event lastTimestamp: "2026-07-02T14:30:28Z" message: 'Created container: model-chainer-raw-hpa-130af' metadata: creationTimestamp: "2026-07-02T14:30:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:28Z" name: model-chainer-raw-hpa-130af-6c866bc956-szxj6.18be7f71883d2bae namespace: kserve-ci-e2e-test resourceVersion: "17213" uid: bc5c02f9-1fcf-47d1-a7e0-40b00fded2f2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-70.ec2.internal source: component: kubelet host: ip-10-0-136-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-130af} kind: Pod name: model-chainer-raw-hpa-130af-6c866bc956-szxj6 namespace: kserve-ci-e2e-test resourceVersion: "17192" uid: b94d640e-9e9d-4c49-adca-66c3a9934370 kind: Event lastTimestamp: "2026-07-02T14:30:28Z" message: Started container model-chainer-raw-hpa-130af metadata: creationTimestamp: "2026-07-02T14:30:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:28Z" name: model-chainer-raw-hpa-130af-6c866bc956-szxj6.18be7f7188b24223 namespace: kserve-ci-e2e-test resourceVersion: "17214" uid: 7e3ba4b9-e4ad-4591-bdd1-f661c6865054 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-70.ec2.internal source: component: kubelet host: ip-10-0-136-70.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-130af} kind: Pod name: model-chainer-raw-hpa-130af-6c866bc956-szxj6 namespace: kserve-ci-e2e-test resourceVersion: "17192" uid: b94d640e-9e9d-4c49-adca-66c3a9934370 kind: Event lastTimestamp: "2026-07-02T14:30:37Z" message: Stopping container model-chainer-raw-hpa-130af metadata: creationTimestamp: "2026-07-02T14:30:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:30:37Z" name: model-chainer-raw-hpa-130af-6c866bc956-szxj6.18be7f73bf33e5b3 namespace: kserve-ci-e2e-test resourceVersion: "17302" uid: b2d8b1f6-1d6d-47ba-bbf6-17c0fbf43abd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-70.ec2.internal source: component: kubelet host: ip-10-0-136-70.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:30:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-130af} kind: Pod name: model-chainer-raw-hpa-130af-6c866bc956-szxj6 namespace: kserve-ci-e2e-test resourceVersion: "17192" uid: b94d640e-9e9d-4c49-adca-66c3a9934370 kind: Event lastTimestamp: "2026-07-02T14:31:04Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:30:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:31:04Z" name: model-chainer-raw-hpa-130af-6c866bc956-szxj6.18be7f7449f6fd10 namespace: kserve-ci-e2e-test resourceVersion: "17682" uid: ea17d3e5-bd35-4be7-b4fc-3e62d3a7603f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-70.ec2.internal source: component: kubelet host: ip-10-0-136-70.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-130af-6c866bc956 namespace: kserve-ci-e2e-test resourceVersion: "17185" uid: d461e019-2a50-4ead-b742-10194880a181 kind: Event lastTimestamp: "2026-07-02T14:30:27Z" message: 'Created pod: model-chainer-raw-hpa-130af-6c866bc956-szxj6' metadata: creationTimestamp: "2026-07-02T14:30:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:30:27Z" name: model-chainer-raw-hpa-130af-6c866bc956.18be7f7167a04f1d namespace: kserve-ci-e2e-test resourceVersion: "17191" uid: 8ae66a02-9432-4f57-8a13-ae0f8f2f3b6a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-130af namespace: kserve-ci-e2e-test resourceVersion: "17183" uid: c5eef032-e0c5-44dd-af4c-f20c590e189d kind: Event lastTimestamp: "2026-07-02T14:30:27Z" message: Scaled up replica set model-chainer-raw-hpa-130af-6c866bc956 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:30:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:30:27Z" name: model-chainer-raw-hpa-130af.18be7f7166b856fd namespace: kserve-ci-e2e-test resourceVersion: "17187" uid: 0bf91670-90b3-413b-94e1-f235f48169a7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:30:27Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-130af namespace: kserve-ci-e2e-test resourceVersion: "17202" uid: cbd44efe-ea02-42c7-89fd-ca7028f766d3 kind: Event lastTimestamp: "2026-07-02T14:30:27Z" message: InferenceGraph [model-chainer-raw-hpa-130af] is Ready metadata: creationTimestamp: "2026-07-02T14:30:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:30:27Z" name: model-chainer-raw-hpa-130af.18be7f716c738b33 namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: c605b12b-ea9c-41ea-8814-7d01bb048845 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:37:14.092167Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r namespace: kserve-ci-e2e-test resourceVersion: "20725" uid: 74a4a633-302b-4a6b-aaff-b061ec2516a6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r to ip-10-0-132-82.ec2.internal metadata: creationTimestamp: "2026-07-02T14:37:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:37:14Z" name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r.18be7fd01082018f namespace: kserve-ci-e2e-test resourceVersion: "20731" uid: ba6a5678-35f9-4e3a-9b06-aa1ccc8523d2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75fd45cf98-dpdjc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:14Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r namespace: kserve-ci-e2e-test resourceVersion: "20735" uid: 74a4a633-302b-4a6b-aaff-b061ec2516a6 kind: Event lastTimestamp: "2026-07-02T14:37:14Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:37:14Z" name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r.18be7fd02a35c0fe namespace: kserve-ci-e2e-test resourceVersion: "20742" uid: ae465ccd-e1f2-4428-8aaf-f55319098fbe reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r namespace: kserve-ci-e2e-test resourceVersion: "20726" uid: 74a4a633-302b-4a6b-aaff-b061ec2516a6 kind: Event lastTimestamp: "2026-07-02T14:37:14Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8e2b89e7009febc315fad3ff8d06e5aec68b35b7cc22df79daf7c1334f909f5a" already present on machine metadata: creationTimestamp: "2026-07-02T14:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:14Z" name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r.18be7fd02b98c1ab namespace: kserve-ci-e2e-test resourceVersion: "20744" uid: a18bbe17-ae0f-4ae9-9eeb-b5ae01bbfa3a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r namespace: kserve-ci-e2e-test resourceVersion: "20726" uid: 74a4a633-302b-4a6b-aaff-b061ec2516a6 kind: Event lastTimestamp: "2026-07-02T14:37:14Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:14Z" name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r.18be7fd02e2a0002 namespace: kserve-ci-e2e-test resourceVersion: "20745" uid: 6f618f7a-9f6f-42eb-a7d5-b91b9a6d26cc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r namespace: kserve-ci-e2e-test resourceVersion: "20726" uid: 74a4a633-302b-4a6b-aaff-b061ec2516a6 kind: Event lastTimestamp: "2026-07-02T14:37:14Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:14Z" name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r.18be7fd02e998c41 namespace: kserve-ci-e2e-test resourceVersion: "20746" uid: 11021a3c-85ad-4730-9719-7014aae9c91d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r namespace: kserve-ci-e2e-test resourceVersion: "20726" uid: 74a4a633-302b-4a6b-aaff-b061ec2516a6 kind: Event lastTimestamp: "2026-07-02T14:37:18Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:18Z" name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r.18be7fd12be8ba98 namespace: kserve-ci-e2e-test resourceVersion: "20782" uid: ab9bedf0-9a44-48d4-9e84-47c9745e8c18 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r namespace: kserve-ci-e2e-test resourceVersion: "20726" uid: 74a4a633-302b-4a6b-aaff-b061ec2516a6 kind: Event lastTimestamp: "2026-07-02T14:37:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:18Z" name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r.18be7fd12f47fb6c namespace: kserve-ci-e2e-test resourceVersion: "20785" uid: 79520a69-6c94-443f-a5d3-b8aecd3674c7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r namespace: kserve-ci-e2e-test resourceVersion: "20726" uid: 74a4a633-302b-4a6b-aaff-b061ec2516a6 kind: Event lastTimestamp: "2026-07-02T14:37:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:18Z" name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r.18be7fd12fe8571e namespace: kserve-ci-e2e-test resourceVersion: "20788" uid: 27d2b2fa-7a22-4a51-b5d4-6f51584287f8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r namespace: kserve-ci-e2e-test resourceVersion: "20726" uid: 74a4a633-302b-4a6b-aaff-b061ec2516a6 kind: Event lastTimestamp: "2026-07-02T14:37:18Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:18Z" name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r.18be7fd12ff1c1f8 namespace: kserve-ci-e2e-test resourceVersion: "20789" uid: 1015bbde-79ec-4647-94fd-ffe3fe048918 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r namespace: kserve-ci-e2e-test resourceVersion: "20726" uid: 74a4a633-302b-4a6b-aaff-b061ec2516a6 kind: Event lastTimestamp: "2026-07-02T14:37:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:18Z" name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r.18be7fd132a37006 namespace: kserve-ci-e2e-test resourceVersion: "20791" uid: 0c3b07e3-7267-46fb-8c38-19f4b6f00155 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r namespace: kserve-ci-e2e-test resourceVersion: "20726" uid: 74a4a633-302b-4a6b-aaff-b061ec2516a6 kind: Event lastTimestamp: "2026-07-02T14:37:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:37:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:18Z" name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r.18be7fd133130eb7 namespace: kserve-ci-e2e-test resourceVersion: "20792" uid: 96e4e24e-ff75-486e-9f96-659e3ff4ed78 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T14:37:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r namespace: kserve-ci-e2e-test resourceVersion: "20726" uid: 74a4a633-302b-4a6b-aaff-b061ec2516a6 kind: Event lastTimestamp: "2026-07-02T14:38:45Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:37:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:45Z" name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r.18be7fd167e29b31 namespace: kserve-ci-e2e-test resourceVersion: "21427" uid: 865e1a50-f2b9-45a5-b666-4fca159c0d6d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r namespace: kserve-ci-e2e-test resourceVersion: "20726" uid: 74a4a633-302b-4a6b-aaff-b061ec2516a6 kind: Event lastTimestamp: "2026-07-02T14:38:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:44Z" name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r.18be7fe50de97a77 namespace: kserve-ci-e2e-test resourceVersion: "21415" uid: 450b2e1b-4813-42f9-8890-c5a140d8f1ff reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r namespace: kserve-ci-e2e-test resourceVersion: "20726" uid: 74a4a633-302b-4a6b-aaff-b061ec2516a6 kind: Event lastTimestamp: "2026-07-02T14:38:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:44Z" name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r.18be7fe50dea1945 namespace: kserve-ci-e2e-test resourceVersion: "21416" uid: 6b932b6a-0b4d-489a-b084-84b73558d2c4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r namespace: kserve-ci-e2e-test resourceVersion: "20726" uid: 74a4a633-302b-4a6b-aaff-b061ec2516a6 kind: Event lastTimestamp: "2026-07-02T14:38:45Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:45Z" name: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r.18be7fe56e0a8aba namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 0c8474dc-d5cd-4236-bd8e-0154d389e937 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-5fcfc-predictor-6bc69857f namespace: kserve-ci-e2e-test resourceVersion: "20715" uid: bb922406-228d-42eb-b57e-102040242e20 kind: Event lastTimestamp: "2026-07-02T14:37:14Z" message: 'Created pod: raw-sklearn-5fcfc-predictor-6bc69857f-rtl7r' metadata: creationTimestamp: "2026-07-02T14:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:37:14Z" name: raw-sklearn-5fcfc-predictor-6bc69857f.18be7fd010266277 namespace: kserve-ci-e2e-test resourceVersion: "20727" uid: 60ff6c50-f02f-4bd3-9d06-447ac4633863 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-5fcfc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: a89b8e3a-44e1-4e34-8052-e05d4ae4039c kind: Event lastTimestamp: "2026-07-02T14:37:14Z" message: Scaled up replica set raw-sklearn-5fcfc-predictor-6bc69857f from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:37:14Z" name: raw-sklearn-5fcfc-predictor.18be7fd00d0b17a4 namespace: kserve-ci-e2e-test resourceVersion: "20718" uid: 8ddeebcb-97e0-4b9e-8092-3208c383544c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:37:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-5fcfc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20714" uid: e90e9d20-9ea6-4d3c-9efc-2bf2174f20e3 kind: Event lastTimestamp: "2026-07-02T14:38:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:37:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:14Z" name: raw-sklearn-5fcfc-predictor.18be7fd38bf9cd5c namespace: kserve-ci-e2e-test resourceVersion: "21180" uid: 5652dc3d-c1f9-4c29-a981-89835c4942d4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:37:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-5fcfc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20714" uid: e90e9d20-9ea6-4d3c-9efc-2bf2174f20e3 kind: Event lastTimestamp: "2026-07-02T14:38:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:37:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:14Z" name: raw-sklearn-5fcfc-predictor.18be7fd38c6a0150 namespace: kserve-ci-e2e-test resourceVersion: "21181" uid: 3fa59a1f-c8f6-4b85-8c21-808fbdd6a0c0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:38:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-5fcfc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20865" uid: e90e9d20-9ea6-4d3c-9efc-2bf2174f20e3 kind: Event lastTimestamp: "2026-07-02T14:38:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:38:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:44Z" name: raw-sklearn-5fcfc-predictor.18be7fe187c68e41 namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: 432a36f1-45c8-49ed-80fa-00ed9f331de4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:38:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-5fcfc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20865" uid: e90e9d20-9ea6-4d3c-9efc-2bf2174f20e3 kind: Event lastTimestamp: "2026-07-02T14:38:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:38:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:44Z" name: raw-sklearn-5fcfc-predictor.18be7fe1883b5177 namespace: kserve-ci-e2e-test resourceVersion: "21399" uid: 78bb0c5f-616a-46d6-96b4-cf9d3ad0a391 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T14:37:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-5fcfc namespace: kserve-ci-e2e-test resourceVersion: "20704" uid: 48ba6a04-c9da-42d3-930b-b5a4d5b5ad6b kind: Event lastTimestamp: "2026-07-02T14:38:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:38:36Z" name: raw-sklearn-5fcfc.18be7fd00d0bff29 namespace: kserve-ci-e2e-test resourceVersion: "21339" uid: d15c6bee-768e-4188-abdb-6dbd39d41723 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-5fcfc namespace: kserve-ci-e2e-test resourceVersion: "20704" uid: 48ba6a04-c9da-42d3-930b-b5a4d5b5ad6b kind: Event lastTimestamp: "2026-07-02T14:37:14Z" message: 'Failed to update status for InferenceService "raw-sklearn-5fcfc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-5fcfc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:37:14Z" name: raw-sklearn-5fcfc.18be7fd00da4d820 namespace: kserve-ci-e2e-test resourceVersion: "20722" uid: 10ece2da-ceb0-4a24-b9ba-d9fe51bc4642 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-5fcfc namespace: kserve-ci-e2e-test resourceVersion: "21336" uid: 48ba6a04-c9da-42d3-930b-b5a4d5b5ad6b kind: Event lastTimestamp: "2026-07-02T14:38:35Z" message: InferenceService [raw-sklearn-5fcfc] is Ready metadata: creationTimestamp: "2026-07-02T14:38:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:38:35Z" name: raw-sklearn-5fcfc.18be7fe3226b8e8d namespace: kserve-ci-e2e-test resourceVersion: "21338" uid: 9e1956b1-9699-4db6-aced-dcd8b9c7fde7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:38:49.403909Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x namespace: kserve-ci-e2e-test resourceVersion: "21483" uid: 92dcd585-8c92-4b29-b754-5b176b678d73 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x to ip-10-0-132-82.ec2.internal metadata: creationTimestamp: "2026-07-02T14:38:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:38:49Z" name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x.18be7fe64187b794 namespace: kserve-ci-e2e-test resourceVersion: "21487" uid: 0137c1f4-7cd5-4905-9784-42d3cfc3143d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75fd45cf98-dpdjc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:49Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x namespace: kserve-ci-e2e-test resourceVersion: "21491" uid: 92dcd585-8c92-4b29-b754-5b176b678d73 kind: Event lastTimestamp: "2026-07-02T14:38:49Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:38:49Z" name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x.18be7fe65adebc78 namespace: kserve-ci-e2e-test resourceVersion: "21499" uid: 740a9071-db72-4597-8b51-e67f15d3c429 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 92dcd585-8c92-4b29-b754-5b176b678d73 kind: Event lastTimestamp: "2026-07-02T14:38:49Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8e2b89e7009febc315fad3ff8d06e5aec68b35b7cc22df79daf7c1334f909f5a" already present on machine metadata: creationTimestamp: "2026-07-02T14:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:49Z" name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x.18be7fe65c6c1def namespace: kserve-ci-e2e-test resourceVersion: "21501" uid: f4061961-d817-4d13-b9ba-267d0160bd21 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 92dcd585-8c92-4b29-b754-5b176b678d73 kind: Event lastTimestamp: "2026-07-02T14:38:49Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:49Z" name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x.18be7fe66076c64b namespace: kserve-ci-e2e-test resourceVersion: "21502" uid: ffb5e016-0675-43ca-9d0a-edd588dea61b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 92dcd585-8c92-4b29-b754-5b176b678d73 kind: Event lastTimestamp: "2026-07-02T14:38:49Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:49Z" name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x.18be7fe660ea5d51 namespace: kserve-ci-e2e-test resourceVersion: "21503" uid: 3e96ed5f-1062-4fe8-865a-1979a378886f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 92dcd585-8c92-4b29-b754-5b176b678d73 kind: Event lastTimestamp: "2026-07-02T14:38:54Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:38:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:54Z" name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x.18be7fe75e94a4df namespace: kserve-ci-e2e-test resourceVersion: "21541" uid: e85ce272-113f-4842-9755-2b5265097873 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 92dcd585-8c92-4b29-b754-5b176b678d73 kind: Event lastTimestamp: "2026-07-02T14:38:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:38:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:54Z" name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x.18be7fe7623b49fe namespace: kserve-ci-e2e-test resourceVersion: "21545" uid: 154c69b7-148d-4717-8bf8-19779efef726 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 92dcd585-8c92-4b29-b754-5b176b678d73 kind: Event lastTimestamp: "2026-07-02T14:38:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:38:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:54Z" name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x.18be7fe762ae5bf5 namespace: kserve-ci-e2e-test resourceVersion: "21546" uid: 971f99b4-41ca-4043-b89e-0b0e2df8c70b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 92dcd585-8c92-4b29-b754-5b176b678d73 kind: Event lastTimestamp: "2026-07-02T14:38:54Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:38:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:54Z" name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x.18be7fe762b9dd08 namespace: kserve-ci-e2e-test resourceVersion: "21547" uid: a3be0182-a26a-488d-80c3-b470bf07f8bc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 92dcd585-8c92-4b29-b754-5b176b678d73 kind: Event lastTimestamp: "2026-07-02T14:38:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:38:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:54Z" name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x.18be7fe765f83472 namespace: kserve-ci-e2e-test resourceVersion: "21549" uid: 43107be0-0eb6-445d-817d-a887d05ec691 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 92dcd585-8c92-4b29-b754-5b176b678d73 kind: Event lastTimestamp: "2026-07-02T14:38:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:38:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:54Z" name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x.18be7fe766692a23 namespace: kserve-ci-e2e-test resourceVersion: "21550" uid: b960772f-a51b-4c05-bca7-5eacd2e63a2f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T14:38:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 92dcd585-8c92-4b29-b754-5b176b678d73 kind: Event lastTimestamp: "2026-07-02T14:40:22Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:38:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:40:22Z" name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x.18be7fe7d65e4ef5 namespace: kserve-ci-e2e-test resourceVersion: "22198" uid: b1cfc1c2-b604-4eae-af64-27894185da98 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:40:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 92dcd585-8c92-4b29-b754-5b176b678d73 kind: Event lastTimestamp: "2026-07-02T14:40:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:40:19Z" name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x.18be7ffb3d5fdd4e namespace: kserve-ci-e2e-test resourceVersion: "22178" uid: a6b64ed2-db26-47dc-8f5d-81ea22beef46 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:40:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 92dcd585-8c92-4b29-b754-5b176b678d73 kind: Event lastTimestamp: "2026-07-02T14:40:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:40:19Z" name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x.18be7ffb3d60fff2 namespace: kserve-ci-e2e-test resourceVersion: "22180" uid: d7d43b5a-2b71-4e8e-88c1-b2ead4af733c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:40:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 92dcd585-8c92-4b29-b754-5b176b678d73 kind: Event lastTimestamp: "2026-07-02T14:40:22Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:40:22Z" name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x.18be7ffbdc8bd2c7 namespace: kserve-ci-e2e-test resourceVersion: "22199" uid: 4f2c3486-0e58-44eb-8898-cd2c19c2eea6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-82.ec2.internal source: component: kubelet host: ip-10-0-132-82.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc namespace: kserve-ci-e2e-test resourceVersion: "21469" uid: 46664199-078a-439b-b50b-176cc5113626 kind: Event lastTimestamp: "2026-07-02T14:38:49Z" message: 'Created pod: raw-sklearn-runtime-6f2af-predictor-94f4f45dc-f7l8x' metadata: creationTimestamp: "2026-07-02T14:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:49Z" name: raw-sklearn-runtime-6f2af-predictor-94f4f45dc.18be7fe640ddd5ae namespace: kserve-ci-e2e-test resourceVersion: "21484" uid: f71d742f-e98f-4980-8df5-bcbf85118476 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-6f2af-predictor namespace: kserve-ci-e2e-test resourceVersion: "21467" uid: 14c2fbfe-7dbf-4dde-93ff-638d89de32b3 kind: Event lastTimestamp: "2026-07-02T14:38:49Z" message: Scaled up replica set raw-sklearn-runtime-6f2af-predictor-94f4f45dc from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:49Z" name: raw-sklearn-runtime-6f2af-predictor.18be7fe63bf72f94 namespace: kserve-ci-e2e-test resourceVersion: "21470" uid: 917f56aa-3fb5-4592-a93a-7cd3f45c489c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:39:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-6f2af-predictor namespace: kserve-ci-e2e-test resourceVersion: "21468" uid: f66af45e-2fbb-45c9-825a-c35746fcc17f kind: Event lastTimestamp: "2026-07-02T14:39:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:39:19Z" name: raw-sklearn-runtime-6f2af-predictor.18be7fe9ba7f0435 namespace: kserve-ci-e2e-test resourceVersion: "21726" uid: 8377f45a-625d-49c1-8e20-2b0eb3678dc3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:39:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-6f2af-predictor namespace: kserve-ci-e2e-test resourceVersion: "21468" uid: f66af45e-2fbb-45c9-825a-c35746fcc17f kind: Event lastTimestamp: "2026-07-02T14:39:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:39:19Z" name: raw-sklearn-runtime-6f2af-predictor.18be7fe9baf72d2a namespace: kserve-ci-e2e-test resourceVersion: "21727" uid: 1f102c8c-5b22-4481-bab0-881916051bdf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:39:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-6f2af-predictor namespace: kserve-ci-e2e-test resourceVersion: "21626" uid: f66af45e-2fbb-45c9-825a-c35746fcc17f kind: Event lastTimestamp: "2026-07-02T14:40:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:39:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:40:19Z" name: raw-sklearn-runtime-6f2af-predictor.18be7ff0b8d1f0c7 namespace: kserve-ci-e2e-test resourceVersion: "22160" uid: 95b9267f-e399-4a08-9422-1e79039c28b7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:39:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-6f2af-predictor namespace: kserve-ci-e2e-test resourceVersion: "21626" uid: f66af45e-2fbb-45c9-825a-c35746fcc17f kind: Event lastTimestamp: "2026-07-02T14:40:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:39:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:40:19Z" name: raw-sklearn-runtime-6f2af-predictor.18be7ff0b93b3413 namespace: kserve-ci-e2e-test resourceVersion: "22162" uid: e55837e5-f48e-4332-988a-d6dda3ebb173 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T14:38:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-6f2af namespace: kserve-ci-e2e-test resourceVersion: "21460" uid: 5d9c8e61-4226-47fd-b778-3273a52346df kind: Event lastTimestamp: "2026-07-02T14:40:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:40:12Z" name: raw-sklearn-runtime-6f2af.18be7fe63bf885a0 namespace: kserve-ci-e2e-test resourceVersion: "22109" uid: d7d74ae5-63ab-4740-a4db-2371a0d621f2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:38:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-6f2af namespace: kserve-ci-e2e-test resourceVersion: "21460" uid: 5d9c8e61-4226-47fd-b778-3273a52346df kind: Event lastTimestamp: "2026-07-02T14:38:49Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-6f2af": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-6f2af": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:38:49Z" name: raw-sklearn-runtime-6f2af.18be7fe63c99cb60 namespace: kserve-ci-e2e-test resourceVersion: "21493" uid: e572489e-927f-4d00-b8c5-b6532bbf0262 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-6f2af namespace: kserve-ci-e2e-test resourceVersion: "21463" uid: 5d9c8e61-4226-47fd-b778-3273a52346df kind: Event lastTimestamp: "2026-07-02T14:38:49Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-6f2af": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:38:49Z" name: raw-sklearn-runtime-6f2af.18be7fe6439258af namespace: kserve-ci-e2e-test resourceVersion: "21494" uid: 8e879bf6-87ae-47e9-b2f9-449571263831 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:40:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-6f2af namespace: kserve-ci-e2e-test resourceVersion: "22107" uid: 5d9c8e61-4226-47fd-b778-3273a52346df kind: Event lastTimestamp: "2026-07-02T14:40:12Z" message: InferenceService [raw-sklearn-runtime-6f2af] is Ready metadata: creationTimestamp: "2026-07-02T14:40:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:40:12Z" name: raw-sklearn-runtime-6f2af.18be7ff98a8d6a7c namespace: kserve-ci-e2e-test resourceVersion: "22108" uid: 5c7a3ce4-c83e-4e8d-90ba-51f3d68311d0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "22430"