--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:11:14.252461Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-6a2fcd-predictor-7b7df4594c-2qktz namespace: kserve-ci-e2e-test resourceVersion: "20715" uid: 202e6c20-cf3e-49e0-89f4-22a2d426498f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-6a2fcd-predictor-7b7df4594c-2qktz to ip-10-0-130-96.ec2.internal metadata: creationTimestamp: "2026-07-13T19:11:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:11:14Z" name: isvc-init-fail-6a2fcd-predictor-7b7df4594c-2qktz.18c1ef2614d10f4e namespace: kserve-ci-e2e-test resourceVersion: "20721" uid: 7bcb30d7-7455-49d2-9372-335d9f218515 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-645b5db44b-6qs9d source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:14Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-6a2fcd-predictor-7b7df4594c-2qktz namespace: kserve-ci-e2e-test resourceVersion: "20726" uid: 202e6c20-cf3e-49e0-89f4-22a2d426498f kind: Event lastTimestamp: "2026-07-13T19:11:14Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:11:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:11:14Z" name: isvc-init-fail-6a2fcd-predictor-7b7df4594c-2qktz.18c1ef262e714bce namespace: kserve-ci-e2e-test resourceVersion: "20735" uid: 6e59c216-17db-4129-92eb-77d492c4d5f7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:11:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-6a2fcd-predictor-7b7df4594c-2qktz namespace: kserve-ci-e2e-test resourceVersion: "20720" uid: 202e6c20-cf3e-49e0-89f4-22a2d426498f kind: Event lastTimestamp: "2026-07-13T19:11:19Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:38f70cb77701ee00560ecdde553dd84342a4d8f7cb0779aa5b508807d9895bf1" already present on machine metadata: creationTimestamp: "2026-07-13T19:11:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:19Z" name: isvc-init-fail-6a2fcd-predictor-7b7df4594c-2qktz.18c1ef26318d3fbf namespace: kserve-ci-e2e-test resourceVersion: "20772" uid: 38495c70-ea49-4fcf-89b0-9a986ea4de6d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:11:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-6a2fcd-predictor-7b7df4594c-2qktz namespace: kserve-ci-e2e-test resourceVersion: "20720" uid: 202e6c20-cf3e-49e0-89f4-22a2d426498f kind: Event lastTimestamp: "2026-07-13T19:11:19Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-13T19:11:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:19Z" name: isvc-init-fail-6a2fcd-predictor-7b7df4594c-2qktz.18c1ef263466a1b5 namespace: kserve-ci-e2e-test resourceVersion: "20775" uid: bc72d9f2-b7e8-42db-82af-6b0c97165eac reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:11:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-6a2fcd-predictor-7b7df4594c-2qktz namespace: kserve-ci-e2e-test resourceVersion: "20720" uid: 202e6c20-cf3e-49e0-89f4-22a2d426498f kind: Event lastTimestamp: "2026-07-13T19:11:19Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-13T19:11:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:19Z" name: isvc-init-fail-6a2fcd-predictor-7b7df4594c-2qktz.18c1ef2634f8b590 namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: 902a2a26-e1ef-44aa-be06-a818d0554112 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-6a2fcd-predictor-7b7df4594c-2qktz namespace: kserve-ci-e2e-test resourceVersion: "20720" uid: 202e6c20-cf3e-49e0-89f4-22a2d426498f kind: Event lastTimestamp: "2026-07-13T19:11:23Z" message: Back-off restarting failed container storage-initializer in pod isvc-init-fail-6a2fcd-predictor-7b7df4594c-2qktz_kserve-ci-e2e-test(202e6c20-cf3e-49e0-89f4-22a2d426498f) metadata: creationTimestamp: "2026-07-13T19:11:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:23Z" name: isvc-init-fail-6a2fcd-predictor-7b7df4594c-2qktz.18c1ef2823066ac4 namespace: kserve-ci-e2e-test resourceVersion: "20808" uid: 9a28e5ac-e82d-4fa6-a614-f7dec960890d reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-6a2fcd-predictor-7b7df4594c namespace: kserve-ci-e2e-test resourceVersion: "20708" uid: 29cef6e8-dd56-4e9d-a1d0-7944846d1ae7 kind: Event lastTimestamp: "2026-07-13T19:11:14Z" message: 'Created pod: isvc-init-fail-6a2fcd-predictor-7b7df4594c-2qktz' metadata: creationTimestamp: "2026-07-13T19:11:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:11:14Z" name: isvc-init-fail-6a2fcd-predictor-7b7df4594c.18c1ef261357437b namespace: kserve-ci-e2e-test resourceVersion: "20718" uid: d0802e6a-063a-4710-b0bd-a5a184cc906d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-6a2fcd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20706" uid: 83be61dd-2b6b-43e8-8ab6-6a4685b658e2 kind: Event lastTimestamp: "2026-07-13T19:11:14Z" message: Scaled up replica set isvc-init-fail-6a2fcd-predictor-7b7df4594c from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:11:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:11:14Z" name: isvc-init-fail-6a2fcd-predictor.18c1ef260f1b2547 namespace: kserve-ci-e2e-test resourceVersion: "20710" uid: 72644da0-f876-40ee-95cf-2bf41110e51c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-13T19:11:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-6a2fcd namespace: kserve-ci-e2e-test resourceVersion: "20699" uid: 312d4911-1e62-4f69-8e9c-22b40c0ebd15 kind: Event lastTimestamp: "2026-07-13T19:11:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:11:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:11:23Z" name: isvc-init-fail-6a2fcd.18c1ef260ee67aed namespace: kserve-ci-e2e-test resourceVersion: "20814" uid: d5195ec3-5eb7-4499-b6ef-71ce4636158b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-6a2fcd namespace: kserve-ci-e2e-test resourceVersion: "20699" uid: 312d4911-1e62-4f69-8e9c-22b40c0ebd15 kind: Event lastTimestamp: "2026-07-13T19:11:14Z" message: 'Failed to update status for InferenceService "isvc-init-fail-6a2fcd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-6a2fcd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:11:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:11:14Z" name: isvc-init-fail-6a2fcd.18c1ef260f8273a8 namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: 37b1cd9e-0405-48b2-bfba-f26c8424c3dc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-6a2fcd namespace: kserve-ci-e2e-test resourceVersion: "20699" uid: 312d4911-1e62-4f69-8e9c-22b40c0ebd15 kind: Event lastTimestamp: "2026-07-13T19:11:14Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-6a2fcd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:11:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:11:14Z" name: isvc-init-fail-6a2fcd.18c1ef260f82b40d namespace: kserve-ci-e2e-test resourceVersion: "20713" uid: 118e547b-6bdf-42a8-90ec-29e4d21b5529 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:04:59.979538Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z namespace: kserve-ci-e2e-test resourceVersion: "17659" uid: 01dc1117-967a-4c02-a616-a014111171ea kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z to ip-10-0-130-96.ec2.internal metadata: creationTimestamp: "2026-07-13T19:04:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:04:59Z" name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z.18c1eecef0697526 namespace: kserve-ci-e2e-test resourceVersion: "17665" uid: 6398cc78-5a18-46b7-8f73-5d09f7139479 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-645b5db44b-6qs9d source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:05:00Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z namespace: kserve-ci-e2e-test resourceVersion: "17670" uid: 01dc1117-967a-4c02-a616-a014111171ea kind: Event lastTimestamp: "2026-07-13T19:05:00Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:05:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:05:00Z" name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z.18c1eecf0a0cd801 namespace: kserve-ci-e2e-test resourceVersion: "17682" uid: 8890560f-9eaa-45b4-95f0-8b765316bd68 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:05:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z namespace: kserve-ci-e2e-test resourceVersion: "17663" uid: 01dc1117-967a-4c02-a616-a014111171ea kind: Event lastTimestamp: "2026-07-13T19:05:00Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:38f70cb77701ee00560ecdde553dd84342a4d8f7cb0779aa5b508807d9895bf1" already present on machine metadata: creationTimestamp: "2026-07-13T19:05:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:05:00Z" name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z.18c1eecf0cdfff0a namespace: kserve-ci-e2e-test resourceVersion: "17685" uid: 618caf52-6480-48f4-8003-e3bd80e01094 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:05:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z namespace: kserve-ci-e2e-test resourceVersion: "17663" uid: 01dc1117-967a-4c02-a616-a014111171ea kind: Event lastTimestamp: "2026-07-13T19:05:00Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-13T19:05:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:05:00Z" name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z.18c1eecf0f7e0311 namespace: kserve-ci-e2e-test resourceVersion: "17686" uid: 66518c0d-2ff8-4647-b62f-e6b35ffc2b47 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:05:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z namespace: kserve-ci-e2e-test resourceVersion: "17663" uid: 01dc1117-967a-4c02-a616-a014111171ea kind: Event lastTimestamp: "2026-07-13T19:05:00Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-13T19:05:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:05:00Z" name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z.18c1eecf100771f7 namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: bb800533-e411-416d-bdb2-5049a85e2592 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:05:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z namespace: kserve-ci-e2e-test resourceVersion: "17663" uid: 01dc1117-967a-4c02-a616-a014111171ea kind: Event lastTimestamp: "2026-07-13T19:05:04Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1730" already present on machine metadata: creationTimestamp: "2026-07-13T19:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:05:04Z" name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z.18c1eed0041e74b8 namespace: kserve-ci-e2e-test resourceVersion: "17723" uid: 9c682230-62bc-427d-87e8-27e1a4f926fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:05:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z namespace: kserve-ci-e2e-test resourceVersion: "17663" uid: 01dc1117-967a-4c02-a616-a014111171ea kind: Event lastTimestamp: "2026-07-13T19:05:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:05:04Z" name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z.18c1eed00740e504 namespace: kserve-ci-e2e-test resourceVersion: "17725" uid: 113a6be6-f717-4c1d-b4c5-cff4736cf0db reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:05:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z namespace: kserve-ci-e2e-test resourceVersion: "17663" uid: 01dc1117-967a-4c02-a616-a014111171ea kind: Event lastTimestamp: "2026-07-13T19:05:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:05:04Z" name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z.18c1eed007b00f1e namespace: kserve-ci-e2e-test resourceVersion: "17726" uid: fc302830-2620-4453-8f6d-63e753fdfba7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:05:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z namespace: kserve-ci-e2e-test resourceVersion: "17663" uid: 01dc1117-967a-4c02-a616-a014111171ea kind: Event lastTimestamp: "2026-07-13T19:05:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:05:04Z" name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z.18c1eed007b946bb namespace: kserve-ci-e2e-test resourceVersion: "17729" uid: 6f078b2f-3247-4a39-b917-53f856a36c89 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:05:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z namespace: kserve-ci-e2e-test resourceVersion: "17663" uid: 01dc1117-967a-4c02-a616-a014111171ea kind: Event lastTimestamp: "2026-07-13T19:05:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:05:04Z" name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z.18c1eed00a7cd70b namespace: kserve-ci-e2e-test resourceVersion: "17731" uid: 1e02d5d0-ac97-4da0-bd8f-8f031603d90f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:05:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z namespace: kserve-ci-e2e-test resourceVersion: "17663" uid: 01dc1117-967a-4c02-a616-a014111171ea kind: Event lastTimestamp: "2026-07-13T19:05:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:05:04Z" name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z.18c1eed00b45ac1f namespace: kserve-ci-e2e-test resourceVersion: "17732" uid: b3bde87f-6803-4c6a-a0db-dabf55427d49 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:05:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z namespace: kserve-ci-e2e-test resourceVersion: "17663" uid: 01dc1117-967a-4c02-a616-a014111171ea kind: Event lastTimestamp: "2026-07-13T19:05:04Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:4e90f566b342ee88137b4332294c4cef6db04c33dbbb6e9394e58bda8902c06d" already present on machine metadata: creationTimestamp: "2026-07-13T19:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:05:04Z" name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z.18c1eed00b4f8072 namespace: kserve-ci-e2e-test resourceVersion: "17733" uid: dfdd7455-4e6a-4dde-8b3b-0b5a60e28247 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:05:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z namespace: kserve-ci-e2e-test resourceVersion: "17663" uid: 01dc1117-967a-4c02-a616-a014111171ea kind: Event lastTimestamp: "2026-07-13T19:05:04Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-13T19:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:05:04Z" name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z.18c1eed00dbf2d1a namespace: kserve-ci-e2e-test resourceVersion: "17735" uid: 3f00c079-06cd-4a1c-9ca0-45599b047d34 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:05:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z namespace: kserve-ci-e2e-test resourceVersion: "17663" uid: 01dc1117-967a-4c02-a616-a014111171ea kind: Event lastTimestamp: "2026-07-13T19:05:04Z" message: Started container agent metadata: creationTimestamp: "2026-07-13T19:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:05:04Z" name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z.18c1eed00e4b9356 namespace: kserve-ci-e2e-test resourceVersion: "17736" uid: 498bd791-dd15-4db8-bccf-a0432b19191b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-13T19:05:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z namespace: kserve-ci-e2e-test resourceVersion: "17663" uid: 01dc1117-967a-4c02-a616-a014111171ea kind: Event lastTimestamp: "2026-07-13T19:06:42Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:05:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:06:42Z" name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z.18c1eed040271244 namespace: kserve-ci-e2e-test resourceVersion: "18562" uid: 77546238-5b8f-4ea2-936a-8036599f275d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-13T19:05:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z namespace: kserve-ci-e2e-test resourceVersion: "17663" uid: 01dc1117-967a-4c02-a616-a014111171ea kind: Event lastTimestamp: "2026-07-13T19:06:42Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-13T19:05:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:06:42Z" name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z.18c1eed07bf6bd43 namespace: kserve-ci-e2e-test resourceVersion: "18563" uid: 49028054-bf5e-4f39-97df-c9ca6573748d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:06:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z namespace: kserve-ci-e2e-test resourceVersion: "17663" uid: 01dc1117-967a-4c02-a616-a014111171ea kind: Event lastTimestamp: "2026-07-13T19:06:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:06:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:06:35Z" name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z.18c1eee51b299f40 namespace: kserve-ci-e2e-test resourceVersion: "18422" uid: fa4e781d-db2f-42e9-8eb1-619137aba067 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:06:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z namespace: kserve-ci-e2e-test resourceVersion: "17663" uid: 01dc1117-967a-4c02-a616-a014111171ea kind: Event lastTimestamp: "2026-07-13T19:06:35Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-13T19:06:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:06:35Z" name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z.18c1eee51b2d6c7b namespace: kserve-ci-e2e-test resourceVersion: "18425" uid: 92c61a5f-08b5-4838-aabd-b85d5baef2a8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:06:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z namespace: kserve-ci-e2e-test resourceVersion: "17663" uid: 01dc1117-967a-4c02-a616-a014111171ea kind: Event lastTimestamp: "2026-07-13T19:06:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:06:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:06:35Z" name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z.18c1eee51b2e0a67 namespace: kserve-ci-e2e-test resourceVersion: "18427" uid: 428c3794-0ec3-4783-8e74-f664c1b229f1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T19:06:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z namespace: kserve-ci-e2e-test resourceVersion: "17663" uid: 01dc1117-967a-4c02-a616-a014111171ea kind: Event lastTimestamp: "2026-07-13T19:06:52Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:06:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:06:52Z" name: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z.18c1eee5ac3a4064 namespace: kserve-ci-e2e-test resourceVersion: "18640" uid: 031ca20a-a38a-414d-a5c7-d0b8cdce0763 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-2a04c-predictor-644c6759b8 namespace: kserve-ci-e2e-test resourceVersion: "17650" uid: 5bd0f579-b752-473b-8c9c-bfb8d71cbe10 kind: Event lastTimestamp: "2026-07-13T19:04:59Z" message: 'Created pod: isvc-logger-raw-2a04c-predictor-644c6759b8-89t6z' metadata: creationTimestamp: "2026-07-13T19:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:04:59Z" name: isvc-logger-raw-2a04c-predictor-644c6759b8.18c1eeceef9ec7c2 namespace: kserve-ci-e2e-test resourceVersion: "17662" uid: 01507c8b-1166-4151-b249-842c581c1bfe reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-2a04c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17647" uid: ca2fc31b-6bdc-47d6-8680-e6ec50908cc4 kind: Event lastTimestamp: "2026-07-13T19:04:59Z" message: Scaled up replica set isvc-logger-raw-2a04c-predictor-644c6759b8 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:04:59Z" name: isvc-logger-raw-2a04c-predictor.18c1eeceebcc7ade namespace: kserve-ci-e2e-test resourceVersion: "17653" uid: 29c552dc-8639-49cf-91f0-621ba123fb50 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:05:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-2a04c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17649" uid: 77d424df-e496-482c-9c30-14e4ce0a398f kind: Event lastTimestamp: "2026-07-13T19:05:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:05:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:05:44Z" name: isvc-logger-raw-2a04c-predictor.18c1eed26a4851ad namespace: kserve-ci-e2e-test resourceVersion: "18040" uid: 739207ca-fc75-4209-a858-55434daf9872 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:05:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-2a04c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17649" uid: 77d424df-e496-482c-9c30-14e4ce0a398f kind: Event lastTimestamp: "2026-07-13T19:05:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:05:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:05:44Z" name: isvc-logger-raw-2a04c-predictor.18c1eed26abca381 namespace: kserve-ci-e2e-test resourceVersion: "18041" uid: 42e29191-0dba-48a9-9573-8d30f3dc2e9e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T19:05:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-2a04c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17817" uid: 77d424df-e496-482c-9c30-14e4ce0a398f kind: Event lastTimestamp: "2026-07-13T19:06:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:06:30Z" name: isvc-logger-raw-2a04c-predictor.18c1eed5e9f5c305 namespace: kserve-ci-e2e-test resourceVersion: "18365" uid: 40f21946-4ba0-457b-b0dc-aab2f180ab2d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T19:05:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-2a04c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17817" uid: 77d424df-e496-482c-9c30-14e4ce0a398f kind: Event lastTimestamp: "2026-07-13T19:06:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:06:30Z" name: isvc-logger-raw-2a04c-predictor.18c1eed5ea6221a1 namespace: kserve-ci-e2e-test resourceVersion: "18366" uid: 0dabed6c-c8b7-4e07-8503-9c41abc770a2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-13T19:04:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-2a04c namespace: kserve-ci-e2e-test resourceVersion: "17638" uid: 0521c3a1-74f0-4211-9a66-e9280f18d92b kind: Event lastTimestamp: "2026-07-13T19:06:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:06:22Z" name: isvc-logger-raw-2a04c.18c1eeceebde294e namespace: kserve-ci-e2e-test resourceVersion: "18315" uid: 866e9006-203e-4e71-b706-0c939463bded reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:04:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-2a04c namespace: kserve-ci-e2e-test resourceVersion: "17638" uid: 0521c3a1-74f0-4211-9a66-e9280f18d92b kind: Event lastTimestamp: "2026-07-13T19:05:00Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-2a04c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-2a04c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:05:00Z" name: isvc-logger-raw-2a04c.18c1eeceec9ec5e4 namespace: kserve-ci-e2e-test resourceVersion: "17672" uid: 7f716ddc-f493-42b4-8361-85c35dc9ed26 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:05:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-2a04c namespace: kserve-ci-e2e-test resourceVersion: "17642" uid: 0521c3a1-74f0-4211-9a66-e9280f18d92b kind: Event lastTimestamp: "2026-07-13T19:05:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-2a04c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:05:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:05:00Z" name: isvc-logger-raw-2a04c.18c1eecef353f1d7 namespace: kserve-ci-e2e-test resourceVersion: "17673" uid: 43a0af2f-4383-4469-9063-2b9ee4a2e361 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:06:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-2a04c namespace: kserve-ci-e2e-test resourceVersion: "18313" uid: 0521c3a1-74f0-4211-9a66-e9280f18d92b kind: Event lastTimestamp: "2026-07-13T19:06:22Z" message: InferenceService [isvc-logger-raw-2a04c] is Ready metadata: creationTimestamp: "2026-07-13T19:06:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:06:22Z" name: isvc-logger-raw-2a04c.18c1eee234d5d948 namespace: kserve-ci-e2e-test resourceVersion: "18314" uid: 4be60558-4522-4e60-9947-28f8ff635a96 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:09:20.676920Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-985ff8-predictor-67fb957c74-z66gl namespace: kserve-ci-e2e-test resourceVersion: "19728" uid: 727c813c-bf1b-465a-ae4a-b37a12240845 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-985ff8-predictor-67fb957c74-z66gl to ip-10-0-130-96.ec2.internal metadata: creationTimestamp: "2026-07-13T19:09:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:09:20Z" name: isvc-primary-985ff8-predictor-67fb957c74-z66gl.18c1ef0ba32fd072 namespace: kserve-ci-e2e-test resourceVersion: "19733" uid: 2e821b5b-def7-471e-8aa1-8d98136a3a50 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-645b5db44b-6qs9d source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:09:21Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-985ff8-predictor-67fb957c74-z66gl namespace: kserve-ci-e2e-test resourceVersion: "19738" uid: 727c813c-bf1b-465a-ae4a-b37a12240845 kind: Event lastTimestamp: "2026-07-13T19:09:21Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:09:21Z" name: isvc-primary-985ff8-predictor-67fb957c74-z66gl.18c1ef0bbc70c341 namespace: kserve-ci-e2e-test resourceVersion: "19747" uid: db319a13-b4d6-41f3-95df-91259c5af406 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:09:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-985ff8-predictor-67fb957c74-z66gl namespace: kserve-ci-e2e-test resourceVersion: "19731" uid: 727c813c-bf1b-465a-ae4a-b37a12240845 kind: Event lastTimestamp: "2026-07-13T19:09:21Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:38f70cb77701ee00560ecdde553dd84342a4d8f7cb0779aa5b508807d9895bf1" already present on machine metadata: creationTimestamp: "2026-07-13T19:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:09:21Z" name: isvc-primary-985ff8-predictor-67fb957c74-z66gl.18c1ef0bbe0d20f6 namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: ee0fb252-597e-4237-a692-c98a7a8d2683 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:09:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-985ff8-predictor-67fb957c74-z66gl namespace: kserve-ci-e2e-test resourceVersion: "19731" uid: 727c813c-bf1b-465a-ae4a-b37a12240845 kind: Event lastTimestamp: "2026-07-13T19:09:21Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-13T19:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:09:21Z" name: isvc-primary-985ff8-predictor-67fb957c74-z66gl.18c1ef0bc1052e4b namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: b3c9e898-9832-446f-ac34-780bde200bc2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:09:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-985ff8-predictor-67fb957c74-z66gl namespace: kserve-ci-e2e-test resourceVersion: "19731" uid: 727c813c-bf1b-465a-ae4a-b37a12240845 kind: Event lastTimestamp: "2026-07-13T19:09:21Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-13T19:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:09:21Z" name: isvc-primary-985ff8-predictor-67fb957c74-z66gl.18c1ef0bc174a958 namespace: kserve-ci-e2e-test resourceVersion: "19751" uid: f48ce3d3-5aec-4b0b-a645-60a70b5d683a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:09:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-985ff8-predictor-67fb957c74-z66gl namespace: kserve-ci-e2e-test resourceVersion: "19731" uid: 727c813c-bf1b-465a-ae4a-b37a12240845 kind: Event lastTimestamp: "2026-07-13T19:09:25Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1730" already present on machine metadata: creationTimestamp: "2026-07-13T19:09:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:09:25Z" name: isvc-primary-985ff8-predictor-67fb957c74-z66gl.18c1ef0cc80c4cdb namespace: kserve-ci-e2e-test resourceVersion: "19796" uid: 62a10909-224b-4eae-b37a-05c554223403 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:09:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-985ff8-predictor-67fb957c74-z66gl namespace: kserve-ci-e2e-test resourceVersion: "19731" uid: 727c813c-bf1b-465a-ae4a-b37a12240845 kind: Event lastTimestamp: "2026-07-13T19:09:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:09:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:09:25Z" name: isvc-primary-985ff8-predictor-67fb957c74-z66gl.18c1ef0ccc348d62 namespace: kserve-ci-e2e-test resourceVersion: "19802" uid: c400c6f4-620a-44b3-ac78-47ebcfc0ca6f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:09:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-985ff8-predictor-67fb957c74-z66gl namespace: kserve-ci-e2e-test resourceVersion: "19731" uid: 727c813c-bf1b-465a-ae4a-b37a12240845 kind: Event lastTimestamp: "2026-07-13T19:09:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:09:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:09:25Z" name: isvc-primary-985ff8-predictor-67fb957c74-z66gl.18c1ef0cccab7f5e namespace: kserve-ci-e2e-test resourceVersion: "19803" uid: 2753da5a-2a28-4537-9a55-35dc765a4238 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:09:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-985ff8-predictor-67fb957c74-z66gl namespace: kserve-ci-e2e-test resourceVersion: "19731" uid: 727c813c-bf1b-465a-ae4a-b37a12240845 kind: Event lastTimestamp: "2026-07-13T19:09:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:09:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:09:25Z" name: isvc-primary-985ff8-predictor-67fb957c74-z66gl.18c1ef0cccb6944c namespace: kserve-ci-e2e-test resourceVersion: "19804" uid: 5752c22c-9062-4300-b534-45a23544e5f6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:09:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-985ff8-predictor-67fb957c74-z66gl namespace: kserve-ci-e2e-test resourceVersion: "19731" uid: 727c813c-bf1b-465a-ae4a-b37a12240845 kind: Event lastTimestamp: "2026-07-13T19:09:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:09:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:09:25Z" name: isvc-primary-985ff8-predictor-67fb957c74-z66gl.18c1ef0ccf87e2d5 namespace: kserve-ci-e2e-test resourceVersion: "19806" uid: ad1886d7-5938-411a-b4f2-f8b2666601c2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:09:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-985ff8-predictor-67fb957c74-z66gl namespace: kserve-ci-e2e-test resourceVersion: "19731" uid: 727c813c-bf1b-465a-ae4a-b37a12240845 kind: Event lastTimestamp: "2026-07-13T19:09:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:09:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:09:25Z" name: isvc-primary-985ff8-predictor-67fb957c74-z66gl.18c1ef0ccffec645 namespace: kserve-ci-e2e-test resourceVersion: "19807" uid: 0c7bcb7e-b780-458d-ae0d-27afb49a64de reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-13T19:09:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-985ff8-predictor-67fb957c74-z66gl namespace: kserve-ci-e2e-test resourceVersion: "19731" uid: 727c813c-bf1b-465a-ae4a-b37a12240845 kind: Event lastTimestamp: "2026-07-13T19:10:38Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:09:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:10:38Z" name: isvc-primary-985ff8-predictor-67fb957c74-z66gl.18c1ef0d042c4294 namespace: kserve-ci-e2e-test resourceVersion: "20316" uid: a300cda4-fb5d-4f7a-9054-5667e7f71124 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-985ff8-predictor-67fb957c74-z66gl namespace: kserve-ci-e2e-test resourceVersion: "19731" uid: 727c813c-bf1b-465a-ae4a-b37a12240845 kind: Event lastTimestamp: "2026-07-13T19:11:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:11:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:09Z" name: isvc-primary-985ff8-predictor-67fb957c74-z66gl.18c1ef24e41bf7a7 namespace: kserve-ci-e2e-test resourceVersion: "20645" uid: a6bdbe2e-0ebe-441b-8ac0-1298fcc43486 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-985ff8-predictor-67fb957c74-z66gl namespace: kserve-ci-e2e-test resourceVersion: "19731" uid: 727c813c-bf1b-465a-ae4a-b37a12240845 kind: Event lastTimestamp: "2026-07-13T19:11:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:11:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:09Z" name: isvc-primary-985ff8-predictor-67fb957c74-z66gl.18c1ef24e41c7ca3 namespace: kserve-ci-e2e-test resourceVersion: "20646" uid: 5ad6052b-7fba-45cf-a1be-be09ae57f7af reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:09:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-985ff8-predictor-67fb957c74 namespace: kserve-ci-e2e-test resourceVersion: "19719" uid: f53d7d53-f90f-4d4e-9367-ed07a0705a1d kind: Event lastTimestamp: "2026-07-13T19:09:20Z" message: 'Created pod: isvc-primary-985ff8-predictor-67fb957c74-z66gl' metadata: creationTimestamp: "2026-07-13T19:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:09:20Z" name: isvc-primary-985ff8-predictor-67fb957c74.18c1ef0ba25bebb8 namespace: kserve-ci-e2e-test resourceVersion: "19729" uid: d686b863-6653-4d37-944d-c530a3635201 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:09:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-985ff8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19717" uid: fb3c501f-87b4-436a-b044-2c5e39f82db7 kind: Event lastTimestamp: "2026-07-13T19:09:20Z" message: Scaled up replica set isvc-primary-985ff8-predictor-67fb957c74 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:09:20Z" name: isvc-primary-985ff8-predictor.18c1ef0b9d6a86d6 namespace: kserve-ci-e2e-test resourceVersion: "19721" uid: a4469563-6292-49a5-87a4-0f58436d17dc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:09:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-985ff8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19718" uid: 62f4561e-d091-4bec-a361-10ee30dce990 kind: Event lastTimestamp: "2026-07-13T19:09:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:09:50Z" name: isvc-primary-985ff8-predictor.18c1ef0f1c72d20c namespace: kserve-ci-e2e-test resourceVersion: "19984" uid: 9eeaf242-33b5-4831-b648-99a60ad4828e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:09:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-985ff8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19718" uid: 62f4561e-d091-4bec-a361-10ee30dce990 kind: Event lastTimestamp: "2026-07-13T19:09:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:09:50Z" name: isvc-primary-985ff8-predictor.18c1ef0f1cd98a04 namespace: kserve-ci-e2e-test resourceVersion: "19985" uid: cc77e67a-584e-4a79-9f34-3a868523beb5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T19:10:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-985ff8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19878" uid: 62f4561e-d091-4bec-a361-10ee30dce990 kind: Event lastTimestamp: "2026-07-13T19:11:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:11:05Z" name: isvc-primary-985ff8-predictor.18c1ef161abd9b9b namespace: kserve-ci-e2e-test resourceVersion: "20582" uid: 9735bb49-f63c-48f8-93a5-58e2c10dd4a4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T19:10:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-985ff8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19878" uid: 62f4561e-d091-4bec-a361-10ee30dce990 kind: Event lastTimestamp: "2026-07-13T19:11:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:11:05Z" name: isvc-primary-985ff8-predictor.18c1ef161b56acb6 namespace: kserve-ci-e2e-test resourceVersion: "20583" uid: 23a2f5b4-d4a5-49f6-9347-26144453ac1b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-13T19:09:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-985ff8 namespace: kserve-ci-e2e-test resourceVersion: "19710" uid: 578e3a62-eb05-4406-ac62-fbc14d43845e kind: Event lastTimestamp: "2026-07-13T19:10:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:10:48Z" name: isvc-primary-985ff8.18c1ef0b9d80daa9 namespace: kserve-ci-e2e-test resourceVersion: "20396" uid: b9bab260-93fb-4b43-8546-256c6f1b4179 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:09:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-985ff8 namespace: kserve-ci-e2e-test resourceVersion: "19710" uid: 578e3a62-eb05-4406-ac62-fbc14d43845e kind: Event lastTimestamp: "2026-07-13T19:09:20Z" message: 'Failed to update status for InferenceService "isvc-primary-985ff8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-985ff8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:09:20Z" name: isvc-primary-985ff8.18c1ef0b9dedaf90 namespace: kserve-ci-e2e-test resourceVersion: "19723" uid: 1cd3872d-cab9-4d97-8439-f9f75482ccb1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:09:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-985ff8 namespace: kserve-ci-e2e-test resourceVersion: "19710" uid: 578e3a62-eb05-4406-ac62-fbc14d43845e kind: Event lastTimestamp: "2026-07-13T19:09:20Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-985ff8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:09:20Z" name: isvc-primary-985ff8.18c1ef0b9deddd7d namespace: kserve-ci-e2e-test resourceVersion: "19725" uid: 017c34ca-143b-45f4-bd8c-eec0feb2f8d0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:10:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-985ff8 namespace: kserve-ci-e2e-test resourceVersion: "20394" uid: 578e3a62-eb05-4406-ac62-fbc14d43845e kind: Event lastTimestamp: "2026-07-13T19:10:48Z" message: InferenceService [isvc-primary-985ff8] is Ready metadata: creationTimestamp: "2026-07-13T19:10:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:10:48Z" name: isvc-primary-985ff8.18c1ef1ffff8b24d namespace: kserve-ci-e2e-test resourceVersion: "20395" uid: 16e7a6cd-dd55-4d61-bd89-a0fa03d046cf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T18:59:23.942130Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6 namespace: kserve-ci-e2e-test resourceVersion: "14580" uid: 97b35062-22c9-418f-854a-24590b93238c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6 to ip-10-0-130-96.ec2.internal metadata: creationTimestamp: "2026-07-13T18:59:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T18:59:23Z" name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6.18c1ee80b305975d namespace: kserve-ci-e2e-test resourceVersion: "14584" uid: 1a30a6f2-4f77-46b8-a230-49bfdb3c6150 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-645b5db44b-6qs9d source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T18:59:24Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 97b35062-22c9-418f-854a-24590b93238c kind: Event lastTimestamp: "2026-07-13T18:59:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-raw-sklearn-batcher-9e27e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T18:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T18:59:24Z" name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6.18c1ee80bd781cb8 namespace: kserve-ci-e2e-test resourceVersion: "14597" uid: 91e61643-a833-412d-83ae-ac343b42d9f0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T18:59:24Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6 namespace: kserve-ci-e2e-test resourceVersion: "14590" uid: 97b35062-22c9-418f-854a-24590b93238c kind: Event lastTimestamp: "2026-07-13T18:59:24Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T18:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T18:59:24Z" name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6.18c1ee80f02afdc2 namespace: kserve-ci-e2e-test resourceVersion: "14609" uid: aa120c22-1abd-43c8-b14f-551b238a6077 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T18:59:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 97b35062-22c9-418f-854a-24590b93238c kind: Event lastTimestamp: "2026-07-13T18:59:24Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:38f70cb77701ee00560ecdde553dd84342a4d8f7cb0779aa5b508807d9895bf1" already present on machine metadata: creationTimestamp: "2026-07-13T18:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T18:59:24Z" name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6.18c1ee80f192783f namespace: kserve-ci-e2e-test resourceVersion: "14611" uid: 6062bd94-3436-4fc9-bb0e-5b04a38f89bf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T18:59:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 97b35062-22c9-418f-854a-24590b93238c kind: Event lastTimestamp: "2026-07-13T18:59:25Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-13T18:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T18:59:25Z" name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6.18c1ee80f42a24d3 namespace: kserve-ci-e2e-test resourceVersion: "14612" uid: 1dc71f30-e068-4ac1-8537-fd6ce415cb78 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T18:59:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 97b35062-22c9-418f-854a-24590b93238c kind: Event lastTimestamp: "2026-07-13T18:59:25Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-13T18:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T18:59:25Z" name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6.18c1ee80f48b28aa namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: 80ffa279-d2a1-41e6-9df1-aaa83bda0db9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T18:59:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 97b35062-22c9-418f-854a-24590b93238c kind: Event lastTimestamp: "2026-07-13T18:59:29Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1730" metadata: creationTimestamp: "2026-07-13T18:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T18:59:29Z" name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6.18c1ee81edce7436 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: cbf6713c-6473-4f0b-b324-2a102a30c28f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T18:59:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 97b35062-22c9-418f-854a-24590b93238c kind: Event lastTimestamp: "2026-07-13T18:59:43Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1730" in 13.939s (13.939s including waiting). Image size: 1555978657 bytes.' metadata: creationTimestamp: "2026-07-13T18:59:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T18:59:43Z" name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6.18c1ee852ca9be58 namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: 91b38794-b4f3-43e0-9fd4-42ddf02f0d49 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T18:59:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 97b35062-22c9-418f-854a-24590b93238c kind: Event lastTimestamp: "2026-07-13T18:59:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T18:59:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T18:59:43Z" name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6.18c1ee8530f16313 namespace: kserve-ci-e2e-test resourceVersion: "14755" uid: 5f143abd-a2e0-4017-9bf0-72e847ddc9a3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T18:59:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 97b35062-22c9-418f-854a-24590b93238c kind: Event lastTimestamp: "2026-07-13T18:59:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T18:59:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T18:59:43Z" name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6.18c1ee853157816c namespace: kserve-ci-e2e-test resourceVersion: "14756" uid: f6813023-718b-418c-aa70-174b6eee20f6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T18:59:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 97b35062-22c9-418f-854a-24590b93238c kind: Event lastTimestamp: "2026-07-13T18:59:43Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-13T18:59:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T18:59:43Z" name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6.18c1ee853163f2d0 namespace: kserve-ci-e2e-test resourceVersion: "14757" uid: a8f54c2d-3ccc-40cd-b5f2-39f4038e4525 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T18:59:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 97b35062-22c9-418f-854a-24590b93238c kind: Event lastTimestamp: "2026-07-13T18:59:45Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.683s (2.683s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-13T18:59:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T18:59:45Z" name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6.18c1ee85d15d889b namespace: kserve-ci-e2e-test resourceVersion: "14777" uid: 6f431b66-86a2-444c-9b2b-def84ef246d3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T18:59:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 97b35062-22c9-418f-854a-24590b93238c kind: Event lastTimestamp: "2026-07-13T18:59:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T18:59:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T18:59:45Z" name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6.18c1ee85d421f1e1 namespace: kserve-ci-e2e-test resourceVersion: "14778" uid: 8cfbef60-10f5-44b9-968c-35f42d2c2381 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T18:59:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 97b35062-22c9-418f-854a-24590b93238c kind: Event lastTimestamp: "2026-07-13T18:59:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T18:59:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T18:59:45Z" name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6.18c1ee85d485aa1d namespace: kserve-ci-e2e-test resourceVersion: "14779" uid: dd907f98-3a4f-472b-91c4-7a1af1424367 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T18:59:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 97b35062-22c9-418f-854a-24590b93238c kind: Event lastTimestamp: "2026-07-13T18:59:45Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:4e90f566b342ee88137b4332294c4cef6db04c33dbbb6e9394e58bda8902c06d" metadata: creationTimestamp: "2026-07-13T18:59:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T18:59:45Z" name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6.18c1ee85d4949efc namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: 58e7bdb7-b4ca-40a4-9b8c-4eca883824c9 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T18:59:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 97b35062-22c9-418f-854a-24590b93238c kind: Event lastTimestamp: "2026-07-13T18:59:49Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:4e90f566b342ee88137b4332294c4cef6db04c33dbbb6e9394e58bda8902c06d" in 3.084s (3.084s including waiting). Image size: 239997016 bytes.' metadata: creationTimestamp: "2026-07-13T18:59:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T18:59:49Z" name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6.18c1ee868c6a5252 namespace: kserve-ci-e2e-test resourceVersion: "14799" uid: e4b355b3-b85f-431e-957a-c464167ea718 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T18:59:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 97b35062-22c9-418f-854a-24590b93238c kind: Event lastTimestamp: "2026-07-13T18:59:49Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-13T18:59:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T18:59:49Z" name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6.18c1ee868f54917c namespace: kserve-ci-e2e-test resourceVersion: "14801" uid: 9f611d54-70ef-4e2a-88f1-3332733aec0e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T18:59:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 97b35062-22c9-418f-854a-24590b93238c kind: Event lastTimestamp: "2026-07-13T18:59:49Z" message: Started container agent metadata: creationTimestamp: "2026-07-13T18:59:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T18:59:49Z" name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6.18c1ee868fbcd459 namespace: kserve-ci-e2e-test resourceVersion: "14802" uid: d6e6996e-b287-4595-966e-6de77d211f63 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-13T18:59:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 97b35062-22c9-418f-854a-24590b93238c kind: Event lastTimestamp: "2026-07-13T19:01:22Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T18:59:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:22Z" name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6.18c1ee86d72dd859 namespace: kserve-ci-e2e-test resourceVersion: "15615" uid: 31b94c3f-3847-497d-bcdc-ee6373e63e27 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-13T18:59:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 97b35062-22c9-418f-854a-24590b93238c kind: Event lastTimestamp: "2026-07-13T19:01:12Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-13T18:59:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:12Z" name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6.18c1ee86d73927c7 namespace: kserve-ci-e2e-test resourceVersion: "15529" uid: b581b0d4-b37c-4c7d-b67e-9b08e1a05a1a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 97b35062-22c9-418f-854a-24590b93238c kind: Event lastTimestamp: "2026-07-13T19:01:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:09Z" name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6.18c1ee992b91a878 namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 0191055d-a182-46f8-8148-698f5dffdae2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 97b35062-22c9-418f-854a-24590b93238c kind: Event lastTimestamp: "2026-07-13T19:01:09Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-13T19:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:09Z" name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6.18c1ee992b91f2ed namespace: kserve-ci-e2e-test resourceVersion: "15411" uid: 57ba28e3-48e5-47d5-ba26-cb0304142e4d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 97b35062-22c9-418f-854a-24590b93238c kind: Event lastTimestamp: "2026-07-13T19:01:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:09Z" name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6.18c1ee992b9271d3 namespace: kserve-ci-e2e-test resourceVersion: "15415" uid: 8de420f5-b334-4b03-b099-2d8f676d46d0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:01:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 97b35062-22c9-418f-854a-24590b93238c kind: Event lastTimestamp: "2026-07-13T19:01:20Z" message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": dial tcp 10.134.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:01:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:20Z" name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6.18c1ee9977830bd0 namespace: kserve-ci-e2e-test resourceVersion: "15602" uid: 8149c92b-7232-4803-83e4-b7b0423326af reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T18:59:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5 namespace: kserve-ci-e2e-test resourceVersion: "14568" uid: e29c01bb-57a9-4bcb-a2e8-4ec709315f8b kind: Event lastTimestamp: "2026-07-13T18:59:23Z" message: 'Created pod: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5-kzcm6' metadata: creationTimestamp: "2026-07-13T18:59:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T18:59:23Z" name: isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5.18c1ee80b2927d57 namespace: kserve-ci-e2e-test resourceVersion: "14581" uid: 7a557042-4072-458d-88ab-827403b68e81 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T18:59:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-9e27e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14567" uid: 2ec80270-0910-46cf-bff5-da63368c213b kind: Event lastTimestamp: "2026-07-13T18:59:23Z" message: Scaled up replica set isvc-raw-sklearn-batcher-9e27e-predictor-54c57f86c5 from 0 to 1 metadata: creationTimestamp: "2026-07-13T18:59:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T18:59:23Z" name: isvc-raw-sklearn-batcher-9e27e-predictor.18c1ee80aaeb43a9 namespace: kserve-ci-e2e-test resourceVersion: "14569" uid: 3c5605f7-35db-425e-810d-d420dc33420a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T18:59:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-9e27e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14576" uid: 8abfc00f-174a-4e9b-a445-0ddee56111f2 kind: Event lastTimestamp: "2026-07-13T19:00:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T18:59:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:00:08Z" name: isvc-raw-sklearn-batcher-9e27e-predictor.18c1ee8430423fa7 namespace: kserve-ci-e2e-test resourceVersion: "14950" uid: fe88c42c-fd9c-4e98-a676-497b0ce8757b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T18:59:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-9e27e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14576" uid: 8abfc00f-174a-4e9b-a445-0ddee56111f2 kind: Event lastTimestamp: "2026-07-13T19:00:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T18:59:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:00:08Z" name: isvc-raw-sklearn-batcher-9e27e-predictor.18c1ee8430a6b6ec namespace: kserve-ci-e2e-test resourceVersion: "14951" uid: 00f747d7-48f6-410a-9105-bc84032a63d0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:00:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-9e27e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14716" uid: 8abfc00f-174a-4e9b-a445-0ddee56111f2 kind: Event lastTimestamp: "2026-07-13T19:00:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:00:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:00:54Z" name: isvc-raw-sklearn-batcher-9e27e-predictor.18c1ee8eae31c322 namespace: kserve-ci-e2e-test resourceVersion: "15275" uid: 8f6be830-a176-4d4a-a9ec-e99593e83ea4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:00:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-9e27e-predictor namespace: kserve-ci-e2e-test resourceVersion: "14716" uid: 8abfc00f-174a-4e9b-a445-0ddee56111f2 kind: Event lastTimestamp: "2026-07-13T19:00:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:00:54Z" name: isvc-raw-sklearn-batcher-9e27e-predictor.18c1ee8eaeb38bb4 namespace: kserve-ci-e2e-test resourceVersion: "15276" uid: 301d36f5-6434-43f9-8e5d-6e965cfa7d08 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-13T18:59:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-9e27e namespace: kserve-ci-e2e-test resourceVersion: "14559" uid: bd139e4f-bd0c-464b-9a5e-98017fbe66f1 kind: Event lastTimestamp: "2026-07-13T19:01:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T18:59:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:01:02Z" name: isvc-raw-sklearn-batcher-9e27e.18c1ee80b0fb7fe3 namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: c465f6d0-f413-4c81-a85c-9dc9058c014f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T18:59:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-9e27e namespace: kserve-ci-e2e-test resourceVersion: "14559" uid: bd139e4f-bd0c-464b-9a5e-98017fbe66f1 kind: Event lastTimestamp: "2026-07-13T18:59:23Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-9e27e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-9e27e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T18:59:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T18:59:23Z" name: isvc-raw-sklearn-batcher-9e27e.18c1ee80b1511b03 namespace: kserve-ci-e2e-test resourceVersion: "14578" uid: 70739ec4-3c22-4bec-8be8-86de3cf4c631 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T18:59:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-9e27e namespace: kserve-ci-e2e-test resourceVersion: "14559" uid: bd139e4f-bd0c-464b-9a5e-98017fbe66f1 kind: Event lastTimestamp: "2026-07-13T18:59:23Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-9e27e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T18:59:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T18:59:23Z" name: isvc-raw-sklearn-batcher-9e27e.18c1ee80b1517cd2 namespace: kserve-ci-e2e-test resourceVersion: "14579" uid: 58a9c6ef-35e2-4b09-8744-d2d31b317d44 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-9e27e namespace: kserve-ci-e2e-test resourceVersion: "15338" uid: bd139e4f-bd0c-464b-9a5e-98017fbe66f1 kind: Event lastTimestamp: "2026-07-13T19:01:02Z" message: InferenceService [isvc-raw-sklearn-batcher-9e27e] is Ready metadata: creationTimestamp: "2026-07-13T19:01:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:01:02Z" name: isvc-raw-sklearn-batcher-9e27e.18c1ee97a06fe815 namespace: kserve-ci-e2e-test resourceVersion: "15339" uid: ac88a9ba-0ec9-4a46-bc8b-bb27e351898c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:10:50.865346Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-985ff8-predictor-8dbc6b87c-9jwt8 namespace: kserve-ci-e2e-test resourceVersion: "20446" uid: 88b86ec4-1a4b-4681-89c4-c3c24b4238ae kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-985ff8-predictor-8dbc6b87c-9jwt8 to ip-10-0-130-96.ec2.internal metadata: creationTimestamp: "2026-07-13T19:10:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:10:50Z" name: isvc-secondary-985ff8-predictor-8dbc6b87c-9jwt8.18c1ef20a2d60b6e namespace: kserve-ci-e2e-test resourceVersion: "20450" uid: 6c15c642-a7f3-4ec5-a849-2623dab806cb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-645b5db44b-6qs9d source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:10:51Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-985ff8-predictor-8dbc6b87c-9jwt8 namespace: kserve-ci-e2e-test resourceVersion: "20455" uid: 88b86ec4-1a4b-4681-89c4-c3c24b4238ae kind: Event lastTimestamp: "2026-07-13T19:10:51Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:10:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:10:51Z" name: isvc-secondary-985ff8-predictor-8dbc6b87c-9jwt8.18c1ef20bbfb3cad namespace: kserve-ci-e2e-test resourceVersion: "20460" uid: f59e9d00-2ef5-4413-a485-c731da873ae1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:10:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-985ff8-predictor-8dbc6b87c-9jwt8 namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: 88b86ec4-1a4b-4681-89c4-c3c24b4238ae kind: Event lastTimestamp: "2026-07-13T19:10:56Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:38f70cb77701ee00560ecdde553dd84342a4d8f7cb0779aa5b508807d9895bf1" already present on machine metadata: creationTimestamp: "2026-07-13T19:10:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:10:56Z" name: isvc-secondary-985ff8-predictor-8dbc6b87c-9jwt8.18c1ef20bd9209a8 namespace: kserve-ci-e2e-test resourceVersion: "20508" uid: 1145a9dc-496f-4b71-a133-2176e74e5541 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:10:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-985ff8-predictor-8dbc6b87c-9jwt8 namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: 88b86ec4-1a4b-4681-89c4-c3c24b4238ae kind: Event lastTimestamp: "2026-07-13T19:10:56Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-13T19:10:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:10:56Z" name: isvc-secondary-985ff8-predictor-8dbc6b87c-9jwt8.18c1ef20c092de40 namespace: kserve-ci-e2e-test resourceVersion: "20510" uid: 18afa108-cc77-4c86-9dfb-2ce5f87d3b3f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:10:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-985ff8-predictor-8dbc6b87c-9jwt8 namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: 88b86ec4-1a4b-4681-89c4-c3c24b4238ae kind: Event lastTimestamp: "2026-07-13T19:10:56Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-13T19:10:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:10:56Z" name: isvc-secondary-985ff8-predictor-8dbc6b87c-9jwt8.18c1ef20c10a6660 namespace: kserve-ci-e2e-test resourceVersion: "20511" uid: f4b8411f-f9dc-4d67-b269-a05aed428a28 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-985ff8-predictor-8dbc6b87c-9jwt8 namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: 88b86ec4-1a4b-4681-89c4-c3c24b4238ae kind: Event lastTimestamp: "2026-07-13T19:11:00Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-985ff8-predictor-8dbc6b87c-9jwt8_kserve-ci-e2e-test(88b86ec4-1a4b-4681-89c4-c3c24b4238ae) metadata: creationTimestamp: "2026-07-13T19:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:00Z" name: isvc-secondary-985ff8-predictor-8dbc6b87c-9jwt8.18c1ef22fd51e47e namespace: kserve-ci-e2e-test resourceVersion: "20545" uid: 5bcb0264-af57-46ca-9bc5-2f069033d30c reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:10:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-985ff8-predictor-8dbc6b87c namespace: kserve-ci-e2e-test resourceVersion: "20435" uid: 19577672-922c-4a85-9560-de95b91dc375 kind: Event lastTimestamp: "2026-07-13T19:10:50Z" message: 'Created pod: isvc-secondary-985ff8-predictor-8dbc6b87c-9jwt8' metadata: creationTimestamp: "2026-07-13T19:10:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:10:50Z" name: isvc-secondary-985ff8-predictor-8dbc6b87c.18c1ef20a2589c03 namespace: kserve-ci-e2e-test resourceVersion: "20447" uid: 4e61b576-1e5e-4ce4-a953-4bf883464447 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:10:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-985ff8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20433" uid: 475b24f0-d327-474d-8881-80923ad67a15 kind: Event lastTimestamp: "2026-07-13T19:10:50Z" message: Scaled up replica set isvc-secondary-985ff8-predictor-8dbc6b87c from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:10:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:10:50Z" name: isvc-secondary-985ff8-predictor.18c1ef209d494c30 namespace: kserve-ci-e2e-test resourceVersion: "20436" uid: 9d28f4de-066a-43f0-9c67-7b546f8ab17a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-985ff8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20434" uid: 26fd5ee8-4696-4aab-bd7a-a3f73953e1d6 kind: Event lastTimestamp: "2026-07-13T19:11:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:11:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:11:05Z" name: isvc-secondary-985ff8-predictor.18c1ef241c3864f3 namespace: kserve-ci-e2e-test resourceVersion: "20585" uid: 2650be67-4754-4ccf-ae02-72dce316cfb0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-985ff8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20434" uid: 26fd5ee8-4696-4aab-bd7a-a3f73953e1d6 kind: Event lastTimestamp: "2026-07-13T19:11:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:11:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:11:05Z" name: isvc-secondary-985ff8-predictor.18c1ef241d414605 namespace: kserve-ci-e2e-test resourceVersion: "20586" uid: 0f02a26d-62e8-473c-aa16-a366a3927063 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-13T19:10:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-985ff8 namespace: kserve-ci-e2e-test resourceVersion: "20426" uid: 1bc15962-e00b-4c82-b8b1-da3bf0504ba3 kind: Event lastTimestamp: "2026-07-13T19:11:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:10:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:11:01Z" name: isvc-secondary-985ff8.18c1ef209d4c45da namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 348933a8-1095-4e76-b937-a1f7969e3dfb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:10:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-985ff8 namespace: kserve-ci-e2e-test resourceVersion: "20426" uid: 1bc15962-e00b-4c82-b8b1-da3bf0504ba3 kind: Event lastTimestamp: "2026-07-13T19:10:50Z" message: 'Failed to update status for InferenceService "isvc-secondary-985ff8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-985ff8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:10:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:10:50Z" name: isvc-secondary-985ff8.18c1ef209dce625a namespace: kserve-ci-e2e-test resourceVersion: "20439" uid: be29889c-033f-45b5-9ca5-6b9b3efc116e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:10:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-985ff8 namespace: kserve-ci-e2e-test resourceVersion: "20426" uid: 1bc15962-e00b-4c82-b8b1-da3bf0504ba3 kind: Event lastTimestamp: "2026-07-13T19:10:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-985ff8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:10:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:10:50Z" name: isvc-secondary-985ff8.18c1ef209dcea311 namespace: kserve-ci-e2e-test resourceVersion: "20440" uid: 11cfdb0d-2603-40ee-849f-bb2850f73498 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:01:09.161399Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5 namespace: kserve-ci-e2e-test resourceVersion: "15431" uid: f1d504da-e581-4fe3-b598-31c5d49d613b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5 to ip-10-0-130-96.ec2.internal metadata: creationTimestamp: "2026-07-13T19:01:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:01:09Z" name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5.18c1ee993294369b namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: 67303c18-4a7b-4d6e-ab6e-a7a99613cdd9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-645b5db44b-6qs9d source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:09Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5 namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: f1d504da-e581-4fe3-b598-31c5d49d613b kind: Event lastTimestamp: "2026-07-13T19:01:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-raw-c7cea-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T19:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:09Z" name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5.18c1ee993afcb305 namespace: kserve-ci-e2e-test resourceVersion: "15473" uid: cf203f98-69bc-4f7b-a9f3-eaa9bded8bc7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:10Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5 namespace: kserve-ci-e2e-test resourceVersion: "15451" uid: f1d504da-e581-4fe3-b598-31c5d49d613b kind: Event lastTimestamp: "2026-07-13T19:01:10Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:01:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:01:10Z" name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5.18c1ee99706f847f namespace: kserve-ci-e2e-test resourceVersion: "15494" uid: 51f7555c-31b2-4705-8620-002fd4a72668 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5 namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: f1d504da-e581-4fe3-b598-31c5d49d613b kind: Event lastTimestamp: "2026-07-13T19:01:10Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:38f70cb77701ee00560ecdde553dd84342a4d8f7cb0779aa5b508807d9895bf1" already present on machine metadata: creationTimestamp: "2026-07-13T19:01:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:10Z" name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5.18c1ee9971e64317 namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: eaf79aae-4c40-4c09-bdc8-7c93fb26708c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5 namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: f1d504da-e581-4fe3-b598-31c5d49d613b kind: Event lastTimestamp: "2026-07-13T19:01:10Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-13T19:01:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:10Z" name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5.18c1ee9974e26d64 namespace: kserve-ci-e2e-test resourceVersion: "15498" uid: b4ec0e0f-65e1-4b8c-a306-8ba6cb9347c3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5 namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: f1d504da-e581-4fe3-b598-31c5d49d613b kind: Event lastTimestamp: "2026-07-13T19:01:10Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-13T19:01:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:10Z" name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5.18c1ee99754186b3 namespace: kserve-ci-e2e-test resourceVersion: "15499" uid: ceaa8016-7764-4fc7-88e3-c7e5bc34795d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5 namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: f1d504da-e581-4fe3-b598-31c5d49d613b kind: Event lastTimestamp: "2026-07-13T19:01:15Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1730" already present on machine metadata: creationTimestamp: "2026-07-13T19:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:15Z" name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5.18c1ee9a9ff29e36 namespace: kserve-ci-e2e-test resourceVersion: "15559" uid: 4999e97f-549c-4838-a946-a65c7531e5c0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5 namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: f1d504da-e581-4fe3-b598-31c5d49d613b kind: Event lastTimestamp: "2026-07-13T19:01:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:15Z" name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5.18c1ee9ac9898984 namespace: kserve-ci-e2e-test resourceVersion: "15566" uid: 2de3b6ed-c108-4748-835d-c928eb057008 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5 namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: f1d504da-e581-4fe3-b598-31c5d49d613b kind: Event lastTimestamp: "2026-07-13T19:01:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:15Z" name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5.18c1ee9ac9ee47c7 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: f7a85d02-82db-41b2-aa3b-caa0f609901f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5 namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: f1d504da-e581-4fe3-b598-31c5d49d613b kind: Event lastTimestamp: "2026-07-13T19:01:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:16Z" name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5.18c1ee9ac9f8e1d8 namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: 6b0ed9c5-e5f6-4301-aea7-b5e6f7c984e1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5 namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: f1d504da-e581-4fe3-b598-31c5d49d613b kind: Event lastTimestamp: "2026-07-13T19:01:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:16Z" name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5.18c1ee9acdd4f69c namespace: kserve-ci-e2e-test resourceVersion: "15569" uid: 40fa0afe-0d07-42e8-96a1-3119f189e428 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5 namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: f1d504da-e581-4fe3-b598-31c5d49d613b kind: Event lastTimestamp: "2026-07-13T19:01:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:16Z" name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5.18c1ee9ace3c2530 namespace: kserve-ci-e2e-test resourceVersion: "15570" uid: 4f548e65-b335-4a08-995f-4f9001a26ed4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-13T19:01:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5 namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: f1d504da-e581-4fe3-b598-31c5d49d613b kind: Event lastTimestamp: "2026-07-13T19:03:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:01:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:03Z" name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5.18c1ee9b2e7b56b5 namespace: kserve-ci-e2e-test resourceVersion: "16578" uid: fbe934bd-6ab8-4022-96e8-3ca1b7219232 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5 namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: f1d504da-e581-4fe3-b598-31c5d49d613b kind: Event lastTimestamp: "2026-07-13T19:02:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:02:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:59Z" name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5.18c1eeb2e92482d5 namespace: kserve-ci-e2e-test resourceVersion: "16440" uid: 911c0a7c-3753-4855-b014-f1397b923d49 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5 namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: f1d504da-e581-4fe3-b598-31c5d49d613b kind: Event lastTimestamp: "2026-07-13T19:02:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:02:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:59Z" name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5.18c1eeb2e9254a9f namespace: kserve-ci-e2e-test resourceVersion: "16441" uid: 44ea0b73-3e87-450f-99df-b199e7466e4b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5 namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: f1d504da-e581-4fe3-b598-31c5d49d613b kind: Event lastTimestamp: "2026-07-13T19:03:03Z" message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp 10.134.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:03:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:03Z" name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5.18c1eeb3dc81009a namespace: kserve-ci-e2e-test resourceVersion: "16577" uid: 32773567-388f-49a0-9049-dba55f2fe44b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47 namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: a364a9a4-831e-452c-a987-434e4e470cd4 kind: Event lastTimestamp: "2026-07-13T19:01:09Z" message: 'Created pod: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47-zk6l5' metadata: creationTimestamp: "2026-07-13T19:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:01:09Z" name: isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47.18c1ee99321ba070 namespace: kserve-ci-e2e-test resourceVersion: "15434" uid: 1049486c-ee41-4f9a-bcc2-8accc3aa6fc0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-c7cea-predictor namespace: kserve-ci-e2e-test resourceVersion: "15418" uid: 27694c83-3558-46f8-86f5-9199f59d27b6 kind: Event lastTimestamp: "2026-07-13T19:01:09Z" message: Scaled up replica set isvc-sklearn-graph-raw-c7cea-predictor-8659c6db47 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:01:09Z" name: isvc-sklearn-graph-raw-c7cea-predictor.18c1ee992d8299e6 namespace: kserve-ci-e2e-test resourceVersion: "15422" uid: b22d5196-10c2-4727-b6c5-e0cf5306c427 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:01:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-c7cea-predictor namespace: kserve-ci-e2e-test resourceVersion: "15419" uid: 388bc934-3eaa-4649-9ec7-042048b2378b kind: Event lastTimestamp: "2026-07-13T19:01:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:01:39Z" name: isvc-sklearn-graph-raw-c7cea-predictor.18c1ee9cac5657c4 namespace: kserve-ci-e2e-test resourceVersion: "15754" uid: 41578ae8-4233-4666-994b-ba9b1482918d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:01:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-c7cea-predictor namespace: kserve-ci-e2e-test resourceVersion: "15419" uid: 388bc934-3eaa-4649-9ec7-042048b2378b kind: Event lastTimestamp: "2026-07-13T19:01:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:01:39Z" name: isvc-sklearn-graph-raw-c7cea-predictor.18c1ee9cacc126fe namespace: kserve-ci-e2e-test resourceVersion: "15755" uid: 59c14f77-e642-47e5-b892-7a0d43436c35 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T19:01:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-c7cea-predictor namespace: kserve-ci-e2e-test resourceVersion: "15633" uid: 388bc934-3eaa-4649-9ec7-042048b2378b kind: Event lastTimestamp: "2026-07-13T19:02:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:01:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:54Z" name: isvc-sklearn-graph-raw-c7cea-predictor.18c1eea3aa96e33b namespace: kserve-ci-e2e-test resourceVersion: "16355" uid: b6451724-94fe-4809-8cb1-73622e38b81a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T19:01:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-c7cea-predictor namespace: kserve-ci-e2e-test resourceVersion: "15633" uid: 388bc934-3eaa-4649-9ec7-042048b2378b kind: Event lastTimestamp: "2026-07-13T19:02:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:01:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:54Z" name: isvc-sklearn-graph-raw-c7cea-predictor.18c1eea3ab013e2a namespace: kserve-ci-e2e-test resourceVersion: "16356" uid: c8766f3a-5a84-4ece-b272-e732ef3841e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-13T19:01:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-c7cea namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: d755c7e6-116d-49ac-82ee-e76ffbd878ec kind: Event lastTimestamp: "2026-07-13T19:02:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:02:33Z" name: isvc-sklearn-graph-raw-c7cea.18c1ee992dbe3436 namespace: kserve-ci-e2e-test resourceVersion: "16162" uid: 0f119f34-a1b9-4b20-81e0-cdbc0f06e863 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-c7cea namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: d755c7e6-116d-49ac-82ee-e76ffbd878ec kind: Event lastTimestamp: "2026-07-13T19:01:09Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-c7cea": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-c7cea": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:01:09Z" name: isvc-sklearn-graph-raw-c7cea.18c1ee992e188017 namespace: kserve-ci-e2e-test resourceVersion: "15425" uid: 26b38719-7a20-4a44-9da3-0fdcb54db805 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-c7cea namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: d755c7e6-116d-49ac-82ee-e76ffbd878ec kind: Event lastTimestamp: "2026-07-13T19:02:33Z" message: InferenceService [isvc-sklearn-graph-raw-c7cea] is Ready metadata: creationTimestamp: "2026-07-13T19:02:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:02:33Z" name: isvc-sklearn-graph-raw-c7cea.18c1eeace56b2933 namespace: kserve-ci-e2e-test resourceVersion: "16161" uid: a8087c96-4b4b-4fe1-b69e-a06a4837e4a1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:02:59.840881Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: 9fc098c6-da5b-4d00-a45d-9e083e7d8406 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw to ip-10-0-130-96.ec2.internal metadata: creationTimestamp: "2026-07-13T19:02:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:02:59Z" name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw.18c1eeb2f7971e4b namespace: kserve-ci-e2e-test resourceVersion: "16491" uid: a614e0f3-8e29-4927-8faf-4abc9a24cd68 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-645b5db44b-6qs9d source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:00Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw namespace: kserve-ci-e2e-test resourceVersion: "16503" uid: 9fc098c6-da5b-4d00-a45d-9e083e7d8406 kind: Event lastTimestamp: "2026-07-13T19:03:00Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:03:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:03:00Z" name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw.18c1eeb310e9b22e namespace: kserve-ci-e2e-test resourceVersion: "16534" uid: 7c6ba162-8458-4736-ae00-529198e5b73d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 9fc098c6-da5b-4d00-a45d-9e083e7d8406 kind: Event lastTimestamp: "2026-07-13T19:03:00Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:38f70cb77701ee00560ecdde553dd84342a4d8f7cb0779aa5b508807d9895bf1" already present on machine metadata: creationTimestamp: "2026-07-13T19:03:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:00Z" name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw.18c1eeb312e3e71c namespace: kserve-ci-e2e-test resourceVersion: "16537" uid: 74a606aa-7fdb-41c9-8b77-f0e29a82daac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 9fc098c6-da5b-4d00-a45d-9e083e7d8406 kind: Event lastTimestamp: "2026-07-13T19:03:00Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-13T19:03:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:00Z" name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw.18c1eeb315d02a0b namespace: kserve-ci-e2e-test resourceVersion: "16538" uid: 37fe4b05-6aeb-4236-a6dc-f59d6f47596b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 9fc098c6-da5b-4d00-a45d-9e083e7d8406 kind: Event lastTimestamp: "2026-07-13T19:03:00Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-13T19:03:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:00Z" name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw.18c1eeb3162c8f4d namespace: kserve-ci-e2e-test resourceVersion: "16539" uid: 3e4fd297-f9d7-45db-9fa4-897b610e3a25 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 9fc098c6-da5b-4d00-a45d-9e083e7d8406 kind: Event lastTimestamp: "2026-07-13T19:03:04Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1730" already present on machine metadata: creationTimestamp: "2026-07-13T19:03:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:04Z" name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw.18c1eeb3f60bf3e8 namespace: kserve-ci-e2e-test resourceVersion: "16583" uid: dca5fa75-f5c7-4f6f-bc06-6a3639e2d5cf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 9fc098c6-da5b-4d00-a45d-9e083e7d8406 kind: Event lastTimestamp: "2026-07-13T19:03:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:03:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:04Z" name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw.18c1eeb3f8ec324a namespace: kserve-ci-e2e-test resourceVersion: "16588" uid: 2e99354d-39f4-4e19-a8a8-e4fe2ff9fca1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 9fc098c6-da5b-4d00-a45d-9e083e7d8406 kind: Event lastTimestamp: "2026-07-13T19:03:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:03:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:04Z" name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw.18c1eeb3f94b2322 namespace: kserve-ci-e2e-test resourceVersion: "16589" uid: 5e0947c3-b493-43cd-acc5-942195bd4cc4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 9fc098c6-da5b-4d00-a45d-9e083e7d8406 kind: Event lastTimestamp: "2026-07-13T19:03:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:03:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:04Z" name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw.18c1eeb3f95445aa namespace: kserve-ci-e2e-test resourceVersion: "16590" uid: f6feb66a-b583-4cb3-8ca6-a17f630b120e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 9fc098c6-da5b-4d00-a45d-9e083e7d8406 kind: Event lastTimestamp: "2026-07-13T19:03:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:03:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:04Z" name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw.18c1eeb3fc6079ae namespace: kserve-ci-e2e-test resourceVersion: "16593" uid: a104bbaf-5ec9-407a-b55b-729cf766ea6d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 9fc098c6-da5b-4d00-a45d-9e083e7d8406 kind: Event lastTimestamp: "2026-07-13T19:03:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:03:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:04Z" name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw.18c1eeb3fd0707ec namespace: kserve-ci-e2e-test resourceVersion: "16594" uid: 4a6261a3-3fb1-4c40-8dd9-aec5e4d80ffd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-13T19:03:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 9fc098c6-da5b-4d00-a45d-9e083e7d8406 kind: Event lastTimestamp: "2026-07-13T19:04:17Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:03:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:04:17Z" name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw.18c1eeb46e50f1fb namespace: kserve-ci-e2e-test resourceVersion: "17167" uid: 851e2f47-dfd3-4eb9-aefb-f63ad215bf9b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 9fc098c6-da5b-4d00-a45d-9e083e7d8406 kind: Event lastTimestamp: "2026-07-13T19:04:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:04:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:04:49Z" name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw.18c1eecc98fc7a9f namespace: kserve-ci-e2e-test resourceVersion: "17500" uid: c987d79f-3a41-4388-910d-f03914e0ddb7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 9fc098c6-da5b-4d00-a45d-9e083e7d8406 kind: Event lastTimestamp: "2026-07-13T19:04:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:04:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:04:49Z" name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw.18c1eecc98fcef92 namespace: kserve-ci-e2e-test resourceVersion: "17502" uid: 55e1710c-06f0-4701-b86d-30de517b0a6b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 9fc098c6-da5b-4d00-a45d-9e083e7d8406 kind: Event lastTimestamp: "2026-07-13T19:04:52Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:04:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:04:52Z" name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw.18c1eecd1ce5db0e namespace: kserve-ci-e2e-test resourceVersion: "17561" uid: 3fbb9568-e862-4a06-833e-63a43efffb04 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c namespace: kserve-ci-e2e-test resourceVersion: "16453" uid: 918e781f-cab1-435e-ba5d-888d6b36c1ba kind: Event lastTimestamp: "2026-07-13T19:02:59Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c-wpvbw' metadata: creationTimestamp: "2026-07-13T19:02:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:59Z" name: isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c.18c1eeb2f6e1b232 namespace: kserve-ci-e2e-test resourceVersion: "16487" uid: a87c605e-b491-4024-a324-f7de12175cfa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-be515-predictor namespace: kserve-ci-e2e-test resourceVersion: "16451" uid: 6f2b5da4-6446-4d41-8aa7-6ef9a6bc3205 kind: Event lastTimestamp: "2026-07-13T19:02:59Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-be515-predictor-7d6cc8854c from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:02:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:59Z" name: isvc-sklearn-graph-raw-hpa-be515-predictor.18c1eeb2ee76b47e namespace: kserve-ci-e2e-test resourceVersion: "16455" uid: a83a292c-c4cc-4e75-8acf-dfe234cb15b2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:03:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-be515-predictor namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: 75ede731-c858-4f11-9761-a62f9e9d11f7 kind: Event lastTimestamp: "2026-07-13T19:03:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:03:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:03:44Z" name: isvc-sklearn-graph-raw-hpa-be515-predictor.18c1eeb66c63e487 namespace: kserve-ci-e2e-test resourceVersion: "16921" uid: b4a8962d-1c9d-4da8-be5b-619c516f5652 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:03:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-be515-predictor namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: 75ede731-c858-4f11-9761-a62f9e9d11f7 kind: Event lastTimestamp: "2026-07-13T19:03:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:03:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:03:44Z" name: isvc-sklearn-graph-raw-hpa-be515-predictor.18c1eeb66cd80ca2 namespace: kserve-ci-e2e-test resourceVersion: "16922" uid: 28a587fe-d5be-4a1d-800e-693502322036 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T19:03:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-be515-predictor namespace: kserve-ci-e2e-test resourceVersion: "16694" uid: 75ede731-c858-4f11-9761-a62f9e9d11f7 kind: Event lastTimestamp: "2026-07-13T19:04:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:03:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:04:44Z" name: isvc-sklearn-graph-raw-hpa-be515-predictor.18c1eeb9ebf6fbd6 namespace: kserve-ci-e2e-test resourceVersion: "17409" uid: dd56a9f7-d533-4213-9b5f-833491e7c135 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T19:03:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-be515-predictor namespace: kserve-ci-e2e-test resourceVersion: "16694" uid: 75ede731-c858-4f11-9761-a62f9e9d11f7 kind: Event lastTimestamp: "2026-07-13T19:04:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:03:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:04:44Z" name: isvc-sklearn-graph-raw-hpa-be515-predictor.18c1eeb9ec85bafb namespace: kserve-ci-e2e-test resourceVersion: "17410" uid: befa313f-0943-4aa8-9edf-fe74ca981d11 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-13T19:02:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-be515 namespace: kserve-ci-e2e-test resourceVersion: "16438" uid: 1381ec38-d5ba-4d94-bb7b-73b885e88460 kind: Event lastTimestamp: "2026-07-13T19:04:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:02:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:04:27Z" name: isvc-sklearn-graph-raw-hpa-be515.18c1eeb2ee838886 namespace: kserve-ci-e2e-test resourceVersion: "17241" uid: 7659213c-357f-4791-8e6a-841519d9f32c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-be515 namespace: kserve-ci-e2e-test resourceVersion: "17239" uid: 1381ec38-d5ba-4d94-bb7b-73b885e88460 kind: Event lastTimestamp: "2026-07-13T19:04:27Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-be515] is Ready metadata: creationTimestamp: "2026-07-13T19:04:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:04:27Z" name: isvc-sklearn-graph-raw-hpa-be515.18c1eec74ac96c04 namespace: kserve-ci-e2e-test resourceVersion: "17240" uid: f9929f06-7455-4ed2-884a-ef427804bb02 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:06:35.361295Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r namespace: kserve-ci-e2e-test resourceVersion: "18463" uid: d41206af-1ae6-4ba5-bf1e-14f15b509f23 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r to ip-10-0-130-96.ec2.internal metadata: creationTimestamp: "2026-07-13T19:06:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:06:35Z" name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r.18c1eee5259ba1bd namespace: kserve-ci-e2e-test resourceVersion: "18467" uid: b0695330-70ce-4a9a-a023-afff8e060cea reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-645b5db44b-6qs9d source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:06:35Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r namespace: kserve-ci-e2e-test resourceVersion: "18465" uid: d41206af-1ae6-4ba5-bf1e-14f15b509f23 kind: Event lastTimestamp: "2026-07-13T19:06:35Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-scale-raw-b1b9c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T19:06:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:06:35Z" name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r.18c1eee531e029d0 namespace: kserve-ci-e2e-test resourceVersion: "18478" uid: 5369ddde-9caa-4e74-8a71-2504898fafb5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:06:36Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r namespace: kserve-ci-e2e-test resourceVersion: "18472" uid: d41206af-1ae6-4ba5-bf1e-14f15b509f23 kind: Event lastTimestamp: "2026-07-13T19:06:36Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:06:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:06:36Z" name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r.18c1eee563ae783e namespace: kserve-ci-e2e-test resourceVersion: "18489" uid: c20cb043-0abd-4510-80d3-c9d3cc31900d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:06:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r namespace: kserve-ci-e2e-test resourceVersion: "18465" uid: d41206af-1ae6-4ba5-bf1e-14f15b509f23 kind: Event lastTimestamp: "2026-07-13T19:06:36Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:38f70cb77701ee00560ecdde553dd84342a4d8f7cb0779aa5b508807d9895bf1" already present on machine metadata: creationTimestamp: "2026-07-13T19:06:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:06:36Z" name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r.18c1eee56546debd namespace: kserve-ci-e2e-test resourceVersion: "18491" uid: 34a824b2-7666-4e12-9798-4ec58e1059ba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:06:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r namespace: kserve-ci-e2e-test resourceVersion: "18465" uid: d41206af-1ae6-4ba5-bf1e-14f15b509f23 kind: Event lastTimestamp: "2026-07-13T19:06:36Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-13T19:06:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:06:36Z" name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r.18c1eee568e2cd07 namespace: kserve-ci-e2e-test resourceVersion: "18492" uid: ff9d506b-7c61-4c88-9365-eef1e3de46a5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:06:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r namespace: kserve-ci-e2e-test resourceVersion: "18465" uid: d41206af-1ae6-4ba5-bf1e-14f15b509f23 kind: Event lastTimestamp: "2026-07-13T19:06:36Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-13T19:06:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:06:36Z" name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r.18c1eee5696e211a namespace: kserve-ci-e2e-test resourceVersion: "18493" uid: 18c57472-75fc-4d3e-8d4f-f4117427fd41 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:06:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r namespace: kserve-ci-e2e-test resourceVersion: "18465" uid: d41206af-1ae6-4ba5-bf1e-14f15b509f23 kind: Event lastTimestamp: "2026-07-13T19:06:40Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1730" already present on machine metadata: creationTimestamp: "2026-07-13T19:06:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:06:40Z" name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r.18c1eee67446cb16 namespace: kserve-ci-e2e-test resourceVersion: "18533" uid: b0584f22-24b7-4c70-a1d4-84881f07a422 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:06:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r namespace: kserve-ci-e2e-test resourceVersion: "18465" uid: d41206af-1ae6-4ba5-bf1e-14f15b509f23 kind: Event lastTimestamp: "2026-07-13T19:06:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:06:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:06:41Z" name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r.18c1eee67761e327 namespace: kserve-ci-e2e-test resourceVersion: "18535" uid: 97c728f7-c1e4-42db-afa5-4e5838875f89 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:06:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r namespace: kserve-ci-e2e-test resourceVersion: "18465" uid: d41206af-1ae6-4ba5-bf1e-14f15b509f23 kind: Event lastTimestamp: "2026-07-13T19:06:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:06:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:06:41Z" name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r.18c1eee677ccd2bb namespace: kserve-ci-e2e-test resourceVersion: "18536" uid: d96ef5bc-28e0-4bd6-8e19-bddfe7dcb67b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:06:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r namespace: kserve-ci-e2e-test resourceVersion: "18465" uid: d41206af-1ae6-4ba5-bf1e-14f15b509f23 kind: Event lastTimestamp: "2026-07-13T19:06:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:06:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:06:41Z" name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r.18c1eee677d75567 namespace: kserve-ci-e2e-test resourceVersion: "18539" uid: bab3826c-7edf-4b1b-87b8-102278a29a6c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:06:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r namespace: kserve-ci-e2e-test resourceVersion: "18465" uid: d41206af-1ae6-4ba5-bf1e-14f15b509f23 kind: Event lastTimestamp: "2026-07-13T19:06:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:06:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:06:41Z" name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r.18c1eee67a836ee9 namespace: kserve-ci-e2e-test resourceVersion: "18541" uid: 8917b6d4-4f49-4fe8-89ce-7d55a549d9a3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:06:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r namespace: kserve-ci-e2e-test resourceVersion: "18465" uid: d41206af-1ae6-4ba5-bf1e-14f15b509f23 kind: Event lastTimestamp: "2026-07-13T19:06:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:06:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:06:41Z" name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r.18c1eee67aed0149 namespace: kserve-ci-e2e-test resourceVersion: "18543" uid: 53f6a128-a251-4113-b5a8-24598a0ae0fa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 19 eventTime: null firstTimestamp: "2026-07-13T19:06:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r namespace: kserve-ci-e2e-test resourceVersion: "18465" uid: d41206af-1ae6-4ba5-bf1e-14f15b509f23 kind: Event lastTimestamp: "2026-07-13T19:09:24Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:06:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:09:24Z" name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r.18c1eee6b04826f7 namespace: kserve-ci-e2e-test resourceVersion: "19780" uid: da0883a5-cad3-4d99-a251-9759323d96dd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:09:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r namespace: kserve-ci-e2e-test resourceVersion: "18465" uid: d41206af-1ae6-4ba5-bf1e-14f15b509f23 kind: Event lastTimestamp: "2026-07-13T19:09:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:09:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:09:15Z" name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r.18c1ef0a6d59bcbc namespace: kserve-ci-e2e-test resourceVersion: "19668" uid: bd5cc492-6333-437c-a8de-a4210fa8c7ab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:09:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r namespace: kserve-ci-e2e-test resourceVersion: "18465" uid: d41206af-1ae6-4ba5-bf1e-14f15b509f23 kind: Event lastTimestamp: "2026-07-13T19:09:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:09:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:09:15Z" name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r.18c1ef0a6d5a543b namespace: kserve-ci-e2e-test resourceVersion: "19669" uid: 9f1acda0-815a-4b0d-813b-37a8cf392ca1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:09:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r namespace: kserve-ci-e2e-test resourceVersion: "18465" uid: d41206af-1ae6-4ba5-bf1e-14f15b509f23 kind: Event lastTimestamp: "2026-07-13T19:09:22Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:09:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:09:22Z" name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r.18c1ef0b02bd9d7b namespace: kserve-ci-e2e-test resourceVersion: "19770" uid: fbdaa2de-7be2-4fcb-bb81-f1b305b7b6ae reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:06:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589 namespace: kserve-ci-e2e-test resourceVersion: "18445" uid: d6078ddc-6dd5-4ebc-8dd7-0de40bcb2185 kind: Event lastTimestamp: "2026-07-13T19:06:35Z" message: 'Created pod: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589-hr65r' metadata: creationTimestamp: "2026-07-13T19:06:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:06:35Z" name: isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589.18c1eee5252ed01f namespace: kserve-ci-e2e-test resourceVersion: "18464" uid: c448237a-8ddd-4e97-a384-d9aafa445094 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:06:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-b1b9c-predictor namespace: kserve-ci-e2e-test resourceVersion: "18439" uid: 7149301b-c530-4d22-9351-3608ef0cda6b kind: Event lastTimestamp: "2026-07-13T19:06:35Z" message: Scaled up replica set isvc-sklearn-scale-raw-b1b9c-predictor-75fb4df589 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:06:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:06:35Z" name: isvc-sklearn-scale-raw-b1b9c-predictor.18c1eee5200f9a69 namespace: kserve-ci-e2e-test resourceVersion: "18449" uid: fd976345-19e9-4d91-98ce-509d0af64488 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:06:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-b1b9c-predictor namespace: kserve-ci-e2e-test resourceVersion: "18443" uid: 0539f610-851e-447f-b0e7-1d4821c077f5 kind: Event lastTimestamp: "2026-07-13T19:07:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:06:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:07:05Z" name: isvc-sklearn-scale-raw-b1b9c-predictor.18c1eee89e88efbf namespace: kserve-ci-e2e-test resourceVersion: "18725" uid: f586c6d4-cc36-40e1-99e6-4fed40cd1aa8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:06:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-b1b9c-predictor namespace: kserve-ci-e2e-test resourceVersion: "18443" uid: 0539f610-851e-447f-b0e7-1d4821c077f5 kind: Event lastTimestamp: "2026-07-13T19:07:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:06:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:07:05Z" name: isvc-sklearn-scale-raw-b1b9c-predictor.18c1eee89efcc899 namespace: kserve-ci-e2e-test resourceVersion: "18726" uid: 901c0528-743e-4289-8a3a-e7a3fdaccafc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-13T19:07:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-b1b9c-predictor namespace: kserve-ci-e2e-test resourceVersion: "18621" uid: 0539f610-851e-447f-b0e7-1d4821c077f5 kind: Event lastTimestamp: "2026-07-13T19:09:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:07:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:09:05Z" name: isvc-sklearn-scale-raw-b1b9c-predictor.18c1eeef9d03e17d namespace: kserve-ci-e2e-test resourceVersion: "19575" uid: e30e86a2-da29-416c-875a-0726b48864c1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-13T19:07:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-b1b9c-predictor namespace: kserve-ci-e2e-test resourceVersion: "18621" uid: 0539f610-851e-447f-b0e7-1d4821c077f5 kind: Event lastTimestamp: "2026-07-13T19:09:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:07:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:09:05Z" name: isvc-sklearn-scale-raw-b1b9c-predictor.18c1eeef9d858fb2 namespace: kserve-ci-e2e-test resourceVersion: "19576" uid: 0fdf9e2c-ceae-4213-ba07-e29a81eeee80 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-13T19:06:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-b1b9c namespace: kserve-ci-e2e-test resourceVersion: "18424" uid: 7110bad7-d518-45fc-8c10-12f34b8a082e kind: Event lastTimestamp: "2026-07-13T19:09:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:06:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:09:14Z" name: isvc-sklearn-scale-raw-b1b9c.18c1eee52013054e namespace: kserve-ci-e2e-test resourceVersion: "19641" uid: 538e4f98-3c9f-48b3-abaa-7de23e2103f0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:06:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-b1b9c namespace: kserve-ci-e2e-test resourceVersion: "18424" uid: 7110bad7-d518-45fc-8c10-12f34b8a082e kind: Event lastTimestamp: "2026-07-13T19:06:35Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-b1b9c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-b1b9c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:06:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:06:35Z" name: isvc-sklearn-scale-raw-b1b9c.18c1eee520a30575 namespace: kserve-ci-e2e-test resourceVersion: "18453" uid: 89837915-5742-49b8-9d94-8d2c2f034cf7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:09:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-b1b9c namespace: kserve-ci-e2e-test resourceVersion: "19638" uid: 7110bad7-d518-45fc-8c10-12f34b8a082e kind: Event lastTimestamp: "2026-07-13T19:09:14Z" message: InferenceService [isvc-sklearn-scale-raw-b1b9c] is Ready metadata: creationTimestamp: "2026-07-13T19:09:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:09:14Z" name: isvc-sklearn-scale-raw-b1b9c.18c1ef0a1ee433bd namespace: kserve-ci-e2e-test resourceVersion: "19640" uid: 9a419679-4174-4f9b-a26e-db95322cee35 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:01:09.236982Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4 namespace: kserve-ci-e2e-test resourceVersion: "15458" uid: 7f985c9a-3629-4e57-8d3e-855799390a54 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4 to ip-10-0-130-96.ec2.internal metadata: creationTimestamp: "2026-07-13T19:01:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:01:09Z" name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4.18c1ee993715882e namespace: kserve-ci-e2e-test resourceVersion: "15463" uid: 9f199ad8-931f-48b2-b928-ac9dfd17f38a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-645b5db44b-6qs9d source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:09Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4 namespace: kserve-ci-e2e-test resourceVersion: "15468" uid: 7f985c9a-3629-4e57-8d3e-855799390a54 kind: Event lastTimestamp: "2026-07-13T19:01:09Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:01:09Z" name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4.18c1ee99507eadb1 namespace: kserve-ci-e2e-test resourceVersion: "15485" uid: e763c333-154e-4e65-95ee-2413bcc87fba reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4 namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: 7f985c9a-3629-4e57-8d3e-855799390a54 kind: Event lastTimestamp: "2026-07-13T19:01:09Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:38f70cb77701ee00560ecdde553dd84342a4d8f7cb0779aa5b508807d9895bf1" already present on machine metadata: creationTimestamp: "2026-07-13T19:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:09Z" name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4.18c1ee9951edaf4a namespace: kserve-ci-e2e-test resourceVersion: "15488" uid: 7beca55e-8ba7-4d86-ac33-9dc206fd8a92 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4 namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: 7f985c9a-3629-4e57-8d3e-855799390a54 kind: Event lastTimestamp: "2026-07-13T19:01:09Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-13T19:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:09Z" name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4.18c1ee995475dac7 namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: e2fec430-6a57-49f1-b95a-4870f3e3b586 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4 namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: 7f985c9a-3629-4e57-8d3e-855799390a54 kind: Event lastTimestamp: "2026-07-13T19:01:09Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-13T19:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:09Z" name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4.18c1ee9954d43470 namespace: kserve-ci-e2e-test resourceVersion: "15491" uid: d0dd1625-affa-4a9b-b80a-b4ab3f17b9e5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4 namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: 7f985c9a-3629-4e57-8d3e-855799390a54 kind: Event lastTimestamp: "2026-07-13T19:01:13Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-13T19:01:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:13Z" name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4.18c1ee9a3e5db49f namespace: kserve-ci-e2e-test resourceVersion: "15539" uid: d0798c73-d249-427a-a710-3c2ad23403b5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4 namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: 7f985c9a-3629-4e57-8d3e-855799390a54 kind: Event lastTimestamp: "2026-07-13T19:01:34Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 20.683s (20.683s including waiting). Image size: 1305567991 bytes.' metadata: creationTimestamp: "2026-07-13T19:01:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:34Z" name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4.18c1ee9f0f37428d namespace: kserve-ci-e2e-test resourceVersion: "15707" uid: 4f084210-3d44-43b1-a6b4-b5bb9552dbee reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4 namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: 7f985c9a-3629-4e57-8d3e-855799390a54 kind: Event lastTimestamp: "2026-07-13T19:01:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:01:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:34Z" name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4.18c1ee9f124c308f namespace: kserve-ci-e2e-test resourceVersion: "15708" uid: 5b91a005-a758-4b37-8254-ead286557e26 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4 namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: 7f985c9a-3629-4e57-8d3e-855799390a54 kind: Event lastTimestamp: "2026-07-13T19:01:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:01:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:34Z" name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4.18c1ee9f12af9df1 namespace: kserve-ci-e2e-test resourceVersion: "15709" uid: e26fb704-6bcb-4eba-9b74-718be0909127 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4 namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: 7f985c9a-3629-4e57-8d3e-855799390a54 kind: Event lastTimestamp: "2026-07-13T19:01:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:01:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:34Z" name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4.18c1ee9f12ba8823 namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: 2551de5a-7f91-42c5-a1de-3c757b20a918 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4 namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: 7f985c9a-3629-4e57-8d3e-855799390a54 kind: Event lastTimestamp: "2026-07-13T19:01:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:01:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:34Z" name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4.18c1ee9f1590bf0f namespace: kserve-ci-e2e-test resourceVersion: "15711" uid: f183d71e-b005-4159-9bba-a41d444164a2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4 namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: 7f985c9a-3629-4e57-8d3e-855799390a54 kind: Event lastTimestamp: "2026-07-13T19:01:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:01:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:34Z" name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4.18c1ee9f15ef8419 namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: d59cdfee-fd26-4eb7-ae32-50c084cb8154 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-13T19:01:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4 namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: 7f985c9a-3629-4e57-8d3e-855799390a54 kind: Event lastTimestamp: "2026-07-13T19:03:00Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:01:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:00Z" name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4.18c1ee9f27fd3fc5 namespace: kserve-ci-e2e-test resourceVersion: "16542" uid: db692c3c-c40b-424f-999f-2faf7fb8f399 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4 namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: 7f985c9a-3629-4e57-8d3e-855799390a54 kind: Event lastTimestamp: "2026-07-13T19:02:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:02:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:59Z" name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4.18c1eeb2f7441f23 namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 1b20cf18-9cd1-487f-be62-e31d9b8d26cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4 namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: 7f985c9a-3629-4e57-8d3e-855799390a54 kind: Event lastTimestamp: "2026-07-13T19:02:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:02:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:59Z" name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4.18c1eeb2f7449d13 namespace: kserve-ci-e2e-test resourceVersion: "16494" uid: 0cf45f50-1938-491c-ac4f-d5735107a8f3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4 namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: 7f985c9a-3629-4e57-8d3e-855799390a54 kind: Event lastTimestamp: "2026-07-13T19:03:00Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:03:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:00Z" name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4.18c1eeb32e33ec90 namespace: kserve-ci-e2e-test resourceVersion: "16541" uid: 067cb3be-c8e4-4baf-b962-53083dc935cd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845 namespace: kserve-ci-e2e-test resourceVersion: "15445" uid: 7b0b9513-ff82-4b29-a9b4-0506e348116e kind: Event lastTimestamp: "2026-07-13T19:01:09Z" message: 'Created pod: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845-4h9q4' metadata: creationTimestamp: "2026-07-13T19:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:01:09Z" name: isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845.18c1ee9936a51ef7 namespace: kserve-ci-e2e-test resourceVersion: "15460" uid: 53360156-82ca-475a-b7ed-0034ca8f1130 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-c7cea-predictor namespace: kserve-ci-e2e-test resourceVersion: "15441" uid: 716253e5-6cba-4a03-a2d3-2e064f19e6d4 kind: Event lastTimestamp: "2026-07-13T19:01:09Z" message: Scaled up replica set isvc-xgboost-graph-raw-c7cea-predictor-86bcb78845 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:01:09Z" name: isvc-xgboost-graph-raw-c7cea-predictor.18c1ee9933eff9e8 namespace: kserve-ci-e2e-test resourceVersion: "15447" uid: ab59e2f7-7801-415d-b418-75c35b3bc260 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:01:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-c7cea-predictor namespace: kserve-ci-e2e-test resourceVersion: "15444" uid: 97d2c68d-0e90-48de-9d31-cb48f12ffbf5 kind: Event lastTimestamp: "2026-07-13T19:02:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:09Z" name: isvc-xgboost-graph-raw-c7cea-predictor.18c1ee9cb2b28953 namespace: kserve-ci-e2e-test resourceVersion: "15980" uid: 74606a5c-f283-4f4c-9dd1-0c22db8b2287 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:01:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-c7cea-predictor namespace: kserve-ci-e2e-test resourceVersion: "15444" uid: 97d2c68d-0e90-48de-9d31-cb48f12ffbf5 kind: Event lastTimestamp: "2026-07-13T19:02:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:09Z" name: isvc-xgboost-graph-raw-c7cea-predictor.18c1ee9cb324f65f namespace: kserve-ci-e2e-test resourceVersion: "15981" uid: d95bc818-2def-4094-813d-108261b11cb3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T19:01:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-c7cea-predictor namespace: kserve-ci-e2e-test resourceVersion: "15636" uid: 97d2c68d-0e90-48de-9d31-cb48f12ffbf5 kind: Event lastTimestamp: "2026-07-13T19:02:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:01:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:54Z" name: isvc-xgboost-graph-raw-c7cea-predictor.18c1eea3b094109b namespace: kserve-ci-e2e-test resourceVersion: "16358" uid: bba8c514-17e0-4969-8ab3-099a2fa46ee0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T19:01:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-c7cea-predictor namespace: kserve-ci-e2e-test resourceVersion: "15636" uid: 97d2c68d-0e90-48de-9d31-cb48f12ffbf5 kind: Event lastTimestamp: "2026-07-13T19:02:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:01:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:54Z" name: isvc-xgboost-graph-raw-c7cea-predictor.18c1eea3b104901f namespace: kserve-ci-e2e-test resourceVersion: "16359" uid: a8aac860-5501-489f-95df-ca4b609e811a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-13T19:01:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-c7cea namespace: kserve-ci-e2e-test resourceVersion: "15427" uid: da1c73c2-b0e9-4717-b2cd-84ac40253897 kind: Event lastTimestamp: "2026-07-13T19:02:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:02:40Z" name: isvc-xgboost-graph-raw-c7cea.18c1ee99341d7e3c namespace: kserve-ci-e2e-test resourceVersion: "16220" uid: 395a95e3-1203-4319-bf3b-d7997bd53366 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-c7cea namespace: kserve-ci-e2e-test resourceVersion: "15427" uid: da1c73c2-b0e9-4717-b2cd-84ac40253897 kind: Event lastTimestamp: "2026-07-13T19:01:09Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-c7cea": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-c7cea": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:01:09Z" name: isvc-xgboost-graph-raw-c7cea.18c1ee99348148d6 namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: 369e7103-793d-460a-876d-849023f56a3e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-c7cea namespace: kserve-ci-e2e-test resourceVersion: "16218" uid: da1c73c2-b0e9-4717-b2cd-84ac40253897 kind: Event lastTimestamp: "2026-07-13T19:02:40Z" message: InferenceService [isvc-xgboost-graph-raw-c7cea] is Ready metadata: creationTimestamp: "2026-07-13T19:02:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:02:40Z" name: isvc-xgboost-graph-raw-c7cea.18c1eeae8d6e2402 namespace: kserve-ci-e2e-test resourceVersion: "16219" uid: 2c28364a-665a-4b2a-85fc-758d5e1396fd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:02:59.956452Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb namespace: kserve-ci-e2e-test resourceVersion: "16507" uid: e07a79b0-93de-4baf-af00-5cfb9a30a665 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb to ip-10-0-130-96.ec2.internal metadata: creationTimestamp: "2026-07-13T19:02:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:02:59Z" name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb.18c1eeb2fe7a8207 namespace: kserve-ci-e2e-test resourceVersion: "16513" uid: 7f8277c6-1f9a-41d8-878a-a2f26f27b03d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-645b5db44b-6qs9d source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:00Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: e07a79b0-93de-4baf-af00-5cfb9a30a665 kind: Event lastTimestamp: "2026-07-13T19:03:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-hpa-be515-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T19:03:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:00Z" name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb.18c1eeb3081d42b5 namespace: kserve-ci-e2e-test resourceVersion: "16527" uid: 568085be-575e-468e-8217-f8fcf68c4e5f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:00Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb namespace: kserve-ci-e2e-test resourceVersion: "16519" uid: e07a79b0-93de-4baf-af00-5cfb9a30a665 kind: Event lastTimestamp: "2026-07-13T19:03:00Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:03:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:03:00Z" name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb.18c1eeb33b6833ec namespace: kserve-ci-e2e-test resourceVersion: "16544" uid: 35fc8619-2440-4af7-bc26-47524cb8908d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: e07a79b0-93de-4baf-af00-5cfb9a30a665 kind: Event lastTimestamp: "2026-07-13T19:03:01Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:38f70cb77701ee00560ecdde553dd84342a4d8f7cb0779aa5b508807d9895bf1" already present on machine metadata: creationTimestamp: "2026-07-13T19:03:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:01Z" name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb.18c1eeb33d1887b2 namespace: kserve-ci-e2e-test resourceVersion: "16546" uid: e7257d9c-9fd4-4c84-a1a7-70b316e21573 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: e07a79b0-93de-4baf-af00-5cfb9a30a665 kind: Event lastTimestamp: "2026-07-13T19:03:01Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-13T19:03:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:01Z" name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb.18c1eeb33fd45615 namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 7741f4cd-b301-457e-ab45-7c797918f61c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: e07a79b0-93de-4baf-af00-5cfb9a30a665 kind: Event lastTimestamp: "2026-07-13T19:03:01Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-13T19:03:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:01Z" name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb.18c1eeb34035193a namespace: kserve-ci-e2e-test resourceVersion: "16549" uid: 263919ef-3876-479f-98b9-71222561d5a3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: e07a79b0-93de-4baf-af00-5cfb9a30a665 kind: Event lastTimestamp: "2026-07-13T19:03:05Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-07-13T19:03:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:05Z" name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb.18c1eeb431f8c8fe namespace: kserve-ci-e2e-test resourceVersion: "16604" uid: a5764337-ef38-4a79-a5a7-cde160b521ef reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: e07a79b0-93de-4baf-af00-5cfb9a30a665 kind: Event lastTimestamp: "2026-07-13T19:03:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:03:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:05Z" name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb.18c1eeb43513cad7 namespace: kserve-ci-e2e-test resourceVersion: "16608" uid: cfd41263-d8b1-4581-b94d-9a1fbe331040 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: e07a79b0-93de-4baf-af00-5cfb9a30a665 kind: Event lastTimestamp: "2026-07-13T19:03:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:03:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:05Z" name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb.18c1eeb4357678d4 namespace: kserve-ci-e2e-test resourceVersion: "16610" uid: cae430ed-a45e-42f7-be6c-542b8e9891fb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: e07a79b0-93de-4baf-af00-5cfb9a30a665 kind: Event lastTimestamp: "2026-07-13T19:03:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:03:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:05Z" name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb.18c1eeb43580ada3 namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 9a906d08-4436-41f3-b67f-6fcf6cfa927b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: e07a79b0-93de-4baf-af00-5cfb9a30a665 kind: Event lastTimestamp: "2026-07-13T19:03:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:03:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:05Z" name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb.18c1eeb43850ddc7 namespace: kserve-ci-e2e-test resourceVersion: "16614" uid: edbc6282-91d1-4398-8718-e3f221d20e1e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: e07a79b0-93de-4baf-af00-5cfb9a30a665 kind: Event lastTimestamp: "2026-07-13T19:03:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:03:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:05Z" name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb.18c1eeb438b1cae8 namespace: kserve-ci-e2e-test resourceVersion: "16615" uid: 96c09d5d-3a83-47fd-9bcc-1a48663a39c2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-13T19:03:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: e07a79b0-93de-4baf-af00-5cfb9a30a665 kind: Event lastTimestamp: "2026-07-13T19:04:53Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:03:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:04:53Z" name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb.18c1eeb4aa6fc04f namespace: kserve-ci-e2e-test resourceVersion: "17573" uid: 1a4c29d8-be7a-41cc-9f14-59f1c8f86220 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: e07a79b0-93de-4baf-af00-5cfb9a30a665 kind: Event lastTimestamp: "2026-07-13T19:04:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:04:50Z" name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb.18c1eecca0591085 namespace: kserve-ci-e2e-test resourceVersion: "17533" uid: 30f0a090-acbc-49a4-9666-ce8d27a60117 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: e07a79b0-93de-4baf-af00-5cfb9a30a665 kind: Event lastTimestamp: "2026-07-13T19:04:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:04:50Z" name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb.18c1eecca0596e5f namespace: kserve-ci-e2e-test resourceVersion: "17534" uid: de166c00-51ed-4958-8751-6933659292e5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: e07a79b0-93de-4baf-af00-5cfb9a30a665 kind: Event lastTimestamp: "2026-07-13T19:04:53Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:04:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:04:53Z" name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb.18c1eecd589d451f namespace: kserve-ci-e2e-test resourceVersion: "17572" uid: 030a5ef2-ae83-4995-874a-c9bbbd6c0fa1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f namespace: kserve-ci-e2e-test resourceVersion: "16488" uid: 54b98924-3465-4bc0-a539-68fed0d32e78 kind: Event lastTimestamp: "2026-07-13T19:02:59Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f-jxlvb' metadata: creationTimestamp: "2026-07-13T19:02:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:59Z" name: isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f.18c1eeb2fda0f42a namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: 42cc2375-5c65-4eff-ac58-0e45f29ceae8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-be515-predictor namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: b3210747-f052-4081-9c68-4242050dd3bc kind: Event lastTimestamp: "2026-07-13T19:02:59Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-be515-predictor-85858c546f from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:02:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:59Z" name: isvc-xgboost-graph-raw-hpa-be515-predictor.18c1eeb2f84eb0a4 namespace: kserve-ci-e2e-test resourceVersion: "16496" uid: ffa95e2b-d8d9-4542-a9d5-c5fd97ca9d9b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:03:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-be515-predictor namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: 486a0c2a-3c2e-46db-b3d5-4f220ecd2259 kind: Event lastTimestamp: "2026-07-13T19:03:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:03:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:03:44Z" name: isvc-xgboost-graph-raw-hpa-be515-predictor.18c1eeb675df4065 namespace: kserve-ci-e2e-test resourceVersion: "16924" uid: 6b008f7d-70ed-4d25-ab0a-02f3f1673a9a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:03:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-be515-predictor namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: 486a0c2a-3c2e-46db-b3d5-4f220ecd2259 kind: Event lastTimestamp: "2026-07-13T19:03:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:03:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:03:44Z" name: isvc-xgboost-graph-raw-hpa-be515-predictor.18c1eeb6766e3cc0 namespace: kserve-ci-e2e-test resourceVersion: "16925" uid: 8909e165-19b9-43b0-a23b-88082024bc0b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T19:03:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-be515-predictor namespace: kserve-ci-e2e-test resourceVersion: "16697" uid: 486a0c2a-3c2e-46db-b3d5-4f220ecd2259 kind: Event lastTimestamp: "2026-07-13T19:04:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:03:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:04:44Z" name: isvc-xgboost-graph-raw-hpa-be515-predictor.18c1eeb9f511083b namespace: kserve-ci-e2e-test resourceVersion: "17411" uid: bc0c7a9f-a77b-49fc-b9d0-ffa2df2c9a5b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T19:03:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-be515-predictor namespace: kserve-ci-e2e-test resourceVersion: "16697" uid: 486a0c2a-3c2e-46db-b3d5-4f220ecd2259 kind: Event lastTimestamp: "2026-07-13T19:04:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:03:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:04:44Z" name: isvc-xgboost-graph-raw-hpa-be515-predictor.18c1eeb9f57dfecc namespace: kserve-ci-e2e-test resourceVersion: "17412" uid: f846174b-4ae7-4519-9730-523077c148fe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-13T19:02:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-be515 namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: 2ab9fbde-d354-4cfc-84f5-c8fd573e0fc6 kind: Event lastTimestamp: "2026-07-13T19:04:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:02:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:04:13Z" name: isvc-xgboost-graph-raw-hpa-be515.18c1eeb2f7d5393f namespace: kserve-ci-e2e-test resourceVersion: "17138" uid: 6be854d4-3866-4adc-9978-fe514801b16b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-be515 namespace: kserve-ci-e2e-test resourceVersion: "17136" uid: 2ab9fbde-d354-4cfc-84f5-c8fd573e0fc6 kind: Event lastTimestamp: "2026-07-13T19:04:13Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-be515] is Ready metadata: creationTimestamp: "2026-07-13T19:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:04:13Z" name: isvc-xgboost-graph-raw-hpa-be515.18c1eec40eb1acef namespace: kserve-ci-e2e-test resourceVersion: "17137" uid: 399f18f7-ea15-4bb3-a417-b6a3d2c95514 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:04:49.958337Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-2a04c-predictor-6bfbf69478-hm649 namespace: kserve-ci-e2e-test resourceVersion: "17504" uid: 790fb00a-b1d5-4e14-8186-8e85874a0fdf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-2a04c-predictor-6bfbf69478-hm649 to ip-10-0-130-96.ec2.internal metadata: creationTimestamp: "2026-07-13T19:04:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:04:49Z" name: message-dumper-raw-2a04c-predictor-6bfbf69478-hm649.18c1eecc9b1a2475 namespace: kserve-ci-e2e-test resourceVersion: "17511" uid: 355d9d5d-195e-47cc-aac7-165aa5771d36 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-645b5db44b-6qs9d source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:50Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-2a04c-predictor-6bfbf69478-hm649 namespace: kserve-ci-e2e-test resourceVersion: "17515" uid: 790fb00a-b1d5-4e14-8186-8e85874a0fdf kind: Event lastTimestamp: "2026-07-13T19:04:50Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:04:50Z" name: message-dumper-raw-2a04c-predictor-6bfbf69478-hm649.18c1eeccb45bcbfb namespace: kserve-ci-e2e-test resourceVersion: "17539" uid: b10652e4-bce5-4f91-acc1-950a0cfcd402 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-2a04c-predictor-6bfbf69478-hm649 namespace: kserve-ci-e2e-test resourceVersion: "17507" uid: 790fb00a-b1d5-4e14-8186-8e85874a0fdf kind: Event lastTimestamp: "2026-07-13T19:04:50Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-13T19:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:04:50Z" name: message-dumper-raw-2a04c-predictor-6bfbf69478-hm649.18c1eeccb6020cae namespace: kserve-ci-e2e-test resourceVersion: "17541" uid: e2511f4e-20ee-48ec-8087-9b2a54da64f0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-2a04c-predictor-6bfbf69478-hm649 namespace: kserve-ci-e2e-test resourceVersion: "17507" uid: 790fb00a-b1d5-4e14-8186-8e85874a0fdf kind: Event lastTimestamp: "2026-07-13T19:04:51Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.137s (1.137s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-13T19:04:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:04:51Z" name: message-dumper-raw-2a04c-predictor-6bfbf69478-hm649.18c1eeccf9cfd53b namespace: kserve-ci-e2e-test resourceVersion: "17552" uid: 55f0d834-79f3-484f-bfd1-85580c2c7359 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-2a04c-predictor-6bfbf69478-hm649 namespace: kserve-ci-e2e-test resourceVersion: "17507" uid: 790fb00a-b1d5-4e14-8186-8e85874a0fdf kind: Event lastTimestamp: "2026-07-13T19:04:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:04:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:04:51Z" name: message-dumper-raw-2a04c-predictor-6bfbf69478-hm649.18c1eeccfd015976 namespace: kserve-ci-e2e-test resourceVersion: "17553" uid: e16a5f23-cd96-4cc0-b983-dd4ce531433e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-2a04c-predictor-6bfbf69478-hm649 namespace: kserve-ci-e2e-test resourceVersion: "17507" uid: 790fb00a-b1d5-4e14-8186-8e85874a0fdf kind: Event lastTimestamp: "2026-07-13T19:04:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:04:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:04:51Z" name: message-dumper-raw-2a04c-predictor-6bfbf69478-hm649.18c1eeccfd742734 namespace: kserve-ci-e2e-test resourceVersion: "17554" uid: 1bbdbdb1-41e0-46b8-bac3-e07c2079ad82 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-2a04c-predictor-6bfbf69478-hm649 namespace: kserve-ci-e2e-test resourceVersion: "17507" uid: 790fb00a-b1d5-4e14-8186-8e85874a0fdf kind: Event lastTimestamp: "2026-07-13T19:04:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:04:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:04:51Z" name: message-dumper-raw-2a04c-predictor-6bfbf69478-hm649.18c1eeccfd7e0519 namespace: kserve-ci-e2e-test resourceVersion: "17555" uid: 9aa87442-4441-4c8e-81be-013085c4f12e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-2a04c-predictor-6bfbf69478-hm649 namespace: kserve-ci-e2e-test resourceVersion: "17507" uid: 790fb00a-b1d5-4e14-8186-8e85874a0fdf kind: Event lastTimestamp: "2026-07-13T19:04:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:04:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:04:51Z" name: message-dumper-raw-2a04c-predictor-6bfbf69478-hm649.18c1eecd02453d42 namespace: kserve-ci-e2e-test resourceVersion: "17557" uid: c02ebc14-bbb0-4c17-b5c8-e9fac7f0d4ce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-2a04c-predictor-6bfbf69478-hm649 namespace: kserve-ci-e2e-test resourceVersion: "17507" uid: 790fb00a-b1d5-4e14-8186-8e85874a0fdf kind: Event lastTimestamp: "2026-07-13T19:04:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:04:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:04:51Z" name: message-dumper-raw-2a04c-predictor-6bfbf69478-hm649.18c1eecd02babf46 namespace: kserve-ci-e2e-test resourceVersion: "17558" uid: f7f9db2b-bf51-4965-80b5-f437290c2ed6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:06:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-2a04c-predictor-6bfbf69478-hm649 namespace: kserve-ci-e2e-test resourceVersion: "17507" uid: 790fb00a-b1d5-4e14-8186-8e85874a0fdf kind: Event lastTimestamp: "2026-07-13T19:06:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:06:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:06:35Z" name: message-dumper-raw-2a04c-predictor-6bfbf69478-hm649.18c1eee520cf260f namespace: kserve-ci-e2e-test resourceVersion: "18454" uid: 040a9591-f2d3-46c9-838a-5659da2f221d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:06:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-2a04c-predictor-6bfbf69478-hm649 namespace: kserve-ci-e2e-test resourceVersion: "17507" uid: 790fb00a-b1d5-4e14-8186-8e85874a0fdf kind: Event lastTimestamp: "2026-07-13T19:06:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:06:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:06:35Z" name: message-dumper-raw-2a04c-predictor-6bfbf69478-hm649.18c1eee520d05ae5 namespace: kserve-ci-e2e-test resourceVersion: "18456" uid: a70de59c-ae8f-4e97-9f9e-d5a0cbeb241d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-2a04c-predictor-6bfbf69478 namespace: kserve-ci-e2e-test resourceVersion: "17479" uid: 0962648a-c868-447d-adba-cdfe221b1a1f kind: Event lastTimestamp: "2026-07-13T19:04:49Z" message: 'Created pod: message-dumper-raw-2a04c-predictor-6bfbf69478-hm649' metadata: creationTimestamp: "2026-07-13T19:04:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:04:49Z" name: message-dumper-raw-2a04c-predictor-6bfbf69478.18c1eecc9aa4b72f namespace: kserve-ci-e2e-test resourceVersion: "17508" uid: 4ac113c3-805c-41e0-8091-f221b0e0c969 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-2a04c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17474" uid: 83db1a6d-3086-49ee-a205-e02bab8403a0 kind: Event lastTimestamp: "2026-07-13T19:04:49Z" message: Scaled up replica set message-dumper-raw-2a04c-predictor-6bfbf69478 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:04:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:04:49Z" name: message-dumper-raw-2a04c-predictor.18c1eecc95f30c06 namespace: kserve-ci-e2e-test resourceVersion: "17483" uid: 2a6feb1e-863c-42d1-add6-fbd119a036b1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:05:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-2a04c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17478" uid: 28cf373d-bc43-4acb-bd0e-12d01802bd1f kind: Event lastTimestamp: "2026-07-13T19:05:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:05:19Z" name: message-dumper-raw-2a04c-predictor.18c1eed01526cd47 namespace: kserve-ci-e2e-test resourceVersion: "17856" uid: de73d03e-b28f-42f2-8e35-9923fe85fadb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:05:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-2a04c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17478" uid: 28cf373d-bc43-4acb-bd0e-12d01802bd1f kind: Event lastTimestamp: "2026-07-13T19:05:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:05:19Z" name: message-dumper-raw-2a04c-predictor.18c1eed0159295ba namespace: kserve-ci-e2e-test resourceVersion: "17857" uid: cc58881c-125c-4535-a4aa-0153baeaaabd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:04:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-2a04c namespace: kserve-ci-e2e-test resourceVersion: "17466" uid: 22893a2e-c07e-4834-80ec-0176cddceb2a kind: Event lastTimestamp: "2026-07-13T19:04:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:04:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:04:59Z" name: message-dumper-raw-2a04c.18c1eecc9624d871 namespace: kserve-ci-e2e-test resourceVersion: "17633" uid: 54f0e793-a88b-4fb6-9759-ee94343cbf44 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-2a04c namespace: kserve-ci-e2e-test resourceVersion: "17466" uid: 22893a2e-c07e-4834-80ec-0176cddceb2a kind: Event lastTimestamp: "2026-07-13T19:04:49Z" message: 'Failed to update status for InferenceService "message-dumper-raw-2a04c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-2a04c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:04:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:04:49Z" name: message-dumper-raw-2a04c.18c1eecc968c752e namespace: kserve-ci-e2e-test resourceVersion: "17490" uid: 9fddc5b5-5b95-4f57-a1e7-8d02a7fb3b17 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-2a04c namespace: kserve-ci-e2e-test resourceVersion: "17631" uid: 22893a2e-c07e-4834-80ec-0176cddceb2a kind: Event lastTimestamp: "2026-07-13T19:04:59Z" message: InferenceService [message-dumper-raw-2a04c] is Ready metadata: creationTimestamp: "2026-07-13T19:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:04:59Z" name: message-dumper-raw-2a04c.18c1eecedc1e6fdd namespace: kserve-ci-e2e-test resourceVersion: "17632" uid: a6932afe-b73f-46a6-b146-c5b559283572 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:02:49.186255Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-c7cea-977849987-ptkqv namespace: kserve-ci-e2e-test resourceVersion: "16290" uid: 7a0af325-045f-4ca3-b2ce-bc423141455f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-c7cea-977849987-ptkqv to ip-10-0-131-242.ec2.internal metadata: creationTimestamp: "2026-07-13T19:02:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:02:49Z" name: model-chainer-raw-c7cea-977849987-ptkqv.18c1eeb07c865689 namespace: kserve-ci-e2e-test resourceVersion: "16295" uid: 8269cb3e-f758-4bd1-b9f3-2f096a37abdc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-645b5db44b-6qs9d source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:49Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-c7cea-977849987-ptkqv namespace: kserve-ci-e2e-test resourceVersion: "16293" uid: 7a0af325-045f-4ca3-b2ce-bc423141455f kind: Event lastTimestamp: "2026-07-13T19:02:49Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-c7cea-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T19:02:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:49Z" name: model-chainer-raw-c7cea-977849987-ptkqv.18c1eeb0896ef745 namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: 42141efc-41fc-4bb3-9464-43eb0f5959fa reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-242.ec2.internal source: component: kubelet host: ip-10-0-131-242.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:50Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-c7cea-977849987-ptkqv namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 7a0af325-045f-4ca3-b2ce-bc423141455f kind: Event lastTimestamp: "2026-07-13T19:02:50Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:02:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:02:50Z" name: model-chainer-raw-c7cea-977849987-ptkqv.18c1eeb0ba29810f namespace: kserve-ci-e2e-test resourceVersion: "16314" uid: feb65815-5c72-4b34-9dfd-afb8769ac63a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c7cea} kind: Pod name: model-chainer-raw-c7cea-977849987-ptkqv namespace: kserve-ci-e2e-test resourceVersion: "16293" uid: 7a0af325-045f-4ca3-b2ce-bc423141455f kind: Event lastTimestamp: "2026-07-13T19:02:50Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:814af323b5d1cbdab913eb2692456d1382c8eaf87a5271e4958bd36fa46947bd" metadata: creationTimestamp: "2026-07-13T19:02:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:50Z" name: model-chainer-raw-c7cea-977849987-ptkqv.18c1eeb0bb7bb55a namespace: kserve-ci-e2e-test resourceVersion: "16316" uid: 80cab8ee-eb28-436a-9c9d-8277cce666e8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-242.ec2.internal source: component: kubelet host: ip-10-0-131-242.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c7cea} kind: Pod name: model-chainer-raw-c7cea-977849987-ptkqv namespace: kserve-ci-e2e-test resourceVersion: "16293" uid: 7a0af325-045f-4ca3-b2ce-bc423141455f kind: Event lastTimestamp: "2026-07-13T19:02:52Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:814af323b5d1cbdab913eb2692456d1382c8eaf87a5271e4958bd36fa46947bd" in 2.236s (2.236s including waiting). Image size: 218072665 bytes.' metadata: creationTimestamp: "2026-07-13T19:02:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:52Z" name: model-chainer-raw-c7cea-977849987-ptkqv.18c1eeb140c83fef namespace: kserve-ci-e2e-test resourceVersion: "16335" uid: cd66e095-12fd-4129-822b-899c7d79b47c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-242.ec2.internal source: component: kubelet host: ip-10-0-131-242.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c7cea} kind: Pod name: model-chainer-raw-c7cea-977849987-ptkqv namespace: kserve-ci-e2e-test resourceVersion: "16293" uid: 7a0af325-045f-4ca3-b2ce-bc423141455f kind: Event lastTimestamp: "2026-07-13T19:02:52Z" message: 'Created container: model-chainer-raw-c7cea' metadata: creationTimestamp: "2026-07-13T19:02:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:52Z" name: model-chainer-raw-c7cea-977849987-ptkqv.18c1eeb143a5f1bc namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: e930fbc4-44fc-441b-ad09-109e340291d0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-242.ec2.internal source: component: kubelet host: ip-10-0-131-242.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c7cea} kind: Pod name: model-chainer-raw-c7cea-977849987-ptkqv namespace: kserve-ci-e2e-test resourceVersion: "16293" uid: 7a0af325-045f-4ca3-b2ce-bc423141455f kind: Event lastTimestamp: "2026-07-13T19:02:52Z" message: Started container model-chainer-raw-c7cea metadata: creationTimestamp: "2026-07-13T19:02:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:52Z" name: model-chainer-raw-c7cea-977849987-ptkqv.18c1eeb14437f179 namespace: kserve-ci-e2e-test resourceVersion: "16338" uid: 7adef70b-14e2-4561-a70e-9869079b41d6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-242.ec2.internal source: component: kubelet host: ip-10-0-131-242.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c7cea} kind: Pod name: model-chainer-raw-c7cea-977849987-ptkqv namespace: kserve-ci-e2e-test resourceVersion: "16293" uid: 7a0af325-045f-4ca3-b2ce-bc423141455f kind: Event lastTimestamp: "2026-07-13T19:02:59Z" message: Stopping container model-chainer-raw-c7cea metadata: creationTimestamp: "2026-07-13T19:02:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:59Z" name: model-chainer-raw-c7cea-977849987-ptkqv.18c1eeb2db147f7a namespace: kserve-ci-e2e-test resourceVersion: "16415" uid: 4408fb0a-af53-46f5-9e58-7ce4db4e3ec8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-242.ec2.internal source: component: kubelet host: ip-10-0-131-242.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:03:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c7cea} kind: Pod name: model-chainer-raw-c7cea-977849987-ptkqv namespace: kserve-ci-e2e-test resourceVersion: "16293" uid: 7a0af325-045f-4ca3-b2ce-bc423141455f kind: Event lastTimestamp: "2026-07-13T19:03:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-13T19:03:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:29Z" name: model-chainer-raw-c7cea-977849987-ptkqv.18c1eeb3f7d54f68 namespace: kserve-ci-e2e-test resourceVersion: "16807" uid: 1e4cb2e6-95fc-47ad-bddf-3de148b8c1d2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-242.ec2.internal source: component: kubelet host: ip-10-0-131-242.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-c7cea-977849987 namespace: kserve-ci-e2e-test resourceVersion: "16287" uid: 2ee70216-1e8e-4efe-bddd-29822096b5ed kind: Event lastTimestamp: "2026-07-13T19:02:49Z" message: 'Created pod: model-chainer-raw-c7cea-977849987-ptkqv' metadata: creationTimestamp: "2026-07-13T19:02:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:49Z" name: model-chainer-raw-c7cea-977849987.18c1eeb07befbf23 namespace: kserve-ci-e2e-test resourceVersion: "16292" uid: baf27f48-7062-4062-aff8-a3ba13ed0ed9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-c7cea namespace: kserve-ci-e2e-test resourceVersion: "16285" uid: 4419f9e8-9473-4806-b2ed-16c436568ad0 kind: Event lastTimestamp: "2026-07-13T19:02:49Z" message: Scaled up replica set model-chainer-raw-c7cea-977849987 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:02:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:49Z" name: model-chainer-raw-c7cea.18c1eeb07aeb2185 namespace: kserve-ci-e2e-test resourceVersion: "16288" uid: ae21721e-09f0-4c38-8e4e-3ff2ee08e4af reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:49Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-c7cea namespace: kserve-ci-e2e-test resourceVersion: "16301" uid: ca1c2d79-6832-43ed-98ee-10a197e86843 kind: Event lastTimestamp: "2026-07-13T19:02:49Z" message: InferenceGraph [model-chainer-raw-c7cea] is Ready metadata: creationTimestamp: "2026-07-13T19:02:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:02:49Z" name: model-chainer-raw-c7cea.18c1eeb07fba2a34 namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: aea95e08-654f-4ccb-8187-a6da6cb24853 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:04:39.659063Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-be515-57944f967b-k9xg4 namespace: kserve-ci-e2e-test resourceVersion: "17346" uid: 76551777-0e0f-410e-86a6-689e2cb3f56d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-be515-57944f967b-k9xg4 to ip-10-0-131-242.ec2.internal metadata: creationTimestamp: "2026-07-13T19:04:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:04:39Z" name: model-chainer-raw-hpa-be515-57944f967b-k9xg4.18c1eeca3537a2dc namespace: kserve-ci-e2e-test resourceVersion: "17350" uid: 2dd9d7f2-74d2-43e8-b9e4-a8735548d6da reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-645b5db44b-6qs9d source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:39Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-be515-57944f967b-k9xg4 namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: 76551777-0e0f-410e-86a6-689e2cb3f56d kind: Event lastTimestamp: "2026-07-13T19:04:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-hpa-be515-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T19:04:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:04:39Z" name: model-chainer-raw-hpa-be515-57944f967b-k9xg4.18c1eeca426049e9 namespace: kserve-ci-e2e-test resourceVersion: "17363" uid: 64ad1700-6e2f-40ec-a14c-323f0f1d8ddd reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-242.ec2.internal source: component: kubelet host: ip-10-0-131-242.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:40Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-be515-57944f967b-k9xg4 namespace: kserve-ci-e2e-test resourceVersion: "17353" uid: 76551777-0e0f-410e-86a6-689e2cb3f56d kind: Event lastTimestamp: "2026-07-13T19:04:40Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:04:40Z" name: model-chainer-raw-hpa-be515-57944f967b-k9xg4.18c1eeca72b50d2e namespace: kserve-ci-e2e-test resourceVersion: "17372" uid: 4a649c07-efc8-44e8-99cd-67db67825d4d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-be515} kind: Pod name: model-chainer-raw-hpa-be515-57944f967b-k9xg4 namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: 76551777-0e0f-410e-86a6-689e2cb3f56d kind: Event lastTimestamp: "2026-07-13T19:04:40Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:814af323b5d1cbdab913eb2692456d1382c8eaf87a5271e4958bd36fa46947bd" already present on machine metadata: creationTimestamp: "2026-07-13T19:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:04:40Z" name: model-chainer-raw-hpa-be515-57944f967b-k9xg4.18c1eeca7445cb78 namespace: kserve-ci-e2e-test resourceVersion: "17374" uid: 22fb0e3e-8300-48bd-a60c-59b48d131aab reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-242.ec2.internal source: component: kubelet host: ip-10-0-131-242.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-be515} kind: Pod name: model-chainer-raw-hpa-be515-57944f967b-k9xg4 namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: 76551777-0e0f-410e-86a6-689e2cb3f56d kind: Event lastTimestamp: "2026-07-13T19:04:40Z" message: 'Created container: model-chainer-raw-hpa-be515' metadata: creationTimestamp: "2026-07-13T19:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:04:40Z" name: model-chainer-raw-hpa-be515-57944f967b-k9xg4.18c1eeca77134c6f namespace: kserve-ci-e2e-test resourceVersion: "17375" uid: 6c43d89f-fcd6-466c-b829-267c3d446db5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-242.ec2.internal source: component: kubelet host: ip-10-0-131-242.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-be515} kind: Pod name: model-chainer-raw-hpa-be515-57944f967b-k9xg4 namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: 76551777-0e0f-410e-86a6-689e2cb3f56d kind: Event lastTimestamp: "2026-07-13T19:04:40Z" message: Started container model-chainer-raw-hpa-be515 metadata: creationTimestamp: "2026-07-13T19:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:04:40Z" name: model-chainer-raw-hpa-be515-57944f967b-k9xg4.18c1eeca77854216 namespace: kserve-ci-e2e-test resourceVersion: "17376" uid: 52a9a566-7343-4e9f-b308-21c3fd92a904 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-242.ec2.internal source: component: kubelet host: ip-10-0-131-242.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-be515} kind: Pod name: model-chainer-raw-hpa-be515-57944f967b-k9xg4 namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: 76551777-0e0f-410e-86a6-689e2cb3f56d kind: Event lastTimestamp: "2026-07-13T19:04:49Z" message: Stopping container model-chainer-raw-hpa-be515 metadata: creationTimestamp: "2026-07-13T19:04:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:04:49Z" name: model-chainer-raw-hpa-be515-57944f967b-k9xg4.18c1eecc8cae5ebf namespace: kserve-ci-e2e-test resourceVersion: "17461" uid: 69cee616-a423-4275-890e-3d8d9f9e806c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-242.ec2.internal source: component: kubelet host: ip-10-0-131-242.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:04:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-be515} kind: Pod name: model-chainer-raw-hpa-be515-57944f967b-k9xg4 namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: 76551777-0e0f-410e-86a6-689e2cb3f56d kind: Event lastTimestamp: "2026-07-13T19:05:17Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-13T19:04:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:05:17Z" name: model-chainer-raw-hpa-be515-57944f967b-k9xg4.18c1eecd3ac0ba05 namespace: kserve-ci-e2e-test resourceVersion: "17840" uid: b1b32cdd-0a8b-4dd6-9e42-a6a74e32e917 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-242.ec2.internal source: component: kubelet host: ip-10-0-131-242.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-be515-57944f967b namespace: kserve-ci-e2e-test resourceVersion: "17342" uid: 793ee8dd-0928-42c7-bb3f-cdd2507b5089 kind: Event lastTimestamp: "2026-07-13T19:04:39Z" message: 'Created pod: model-chainer-raw-hpa-be515-57944f967b-k9xg4' metadata: creationTimestamp: "2026-07-13T19:04:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:04:39Z" name: model-chainer-raw-hpa-be515-57944f967b.18c1eeca34bce0f0 namespace: kserve-ci-e2e-test resourceVersion: "17348" uid: 9c9e7df4-7e43-40b7-8d8f-ae457e071774 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-be515 namespace: kserve-ci-e2e-test resourceVersion: "17340" uid: 9439ada5-2e76-4332-9daf-c524bc9f945d kind: Event lastTimestamp: "2026-07-13T19:04:39Z" message: Scaled up replica set model-chainer-raw-hpa-be515-57944f967b from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:04:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:04:39Z" name: model-chainer-raw-hpa-be515.18c1eeca32b5b215 namespace: kserve-ci-e2e-test resourceVersion: "17343" uid: 92856410-db02-43ef-a105-760b2387f60a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:04:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-be515 namespace: kserve-ci-e2e-test resourceVersion: "17356" uid: 62f65de4-d7e5-450c-8d24-66373a837b13 kind: Event lastTimestamp: "2026-07-13T19:04:39Z" message: InferenceGraph [model-chainer-raw-hpa-be515] is Ready metadata: creationTimestamp: "2026-07-13T19:04:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:04:39Z" name: model-chainer-raw-hpa-be515.18c1eeca39832e24 namespace: kserve-ci-e2e-test resourceVersion: "17358" uid: d0981a63-b09b-4b3e-8bda-f5f23532ca88 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:11:29.580803Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7 namespace: kserve-ci-e2e-test resourceVersion: "20904" uid: c9b6e7e1-e413-42e0-84df-4463dd0d9686 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7 to ip-10-0-130-96.ec2.internal metadata: creationTimestamp: "2026-07-13T19:11:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:11:29Z" name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7.18c1ef29a674f616 namespace: kserve-ci-e2e-test resourceVersion: "20912" uid: 55fa3df3-8872-4c6c-9128-0d419b9f082f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-645b5db44b-6qs9d source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:30Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7 namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: c9b6e7e1-e413-42e0-84df-4463dd0d9686 kind: Event lastTimestamp: "2026-07-13T19:11:30Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:11:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:11:30Z" name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7.18c1ef29bfd2a704 namespace: kserve-ci-e2e-test resourceVersion: "20922" uid: f5ed12b4-c51a-4470-9243-dc6808254179 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7 namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: c9b6e7e1-e413-42e0-84df-4463dd0d9686 kind: Event lastTimestamp: "2026-07-13T19:11:30Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:38f70cb77701ee00560ecdde553dd84342a4d8f7cb0779aa5b508807d9895bf1" already present on machine metadata: creationTimestamp: "2026-07-13T19:11:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:30Z" name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7.18c1ef29c1ccc7d0 namespace: kserve-ci-e2e-test resourceVersion: "20924" uid: 352206a9-a8f9-4ef3-a872-b675d997e53b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7 namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: c9b6e7e1-e413-42e0-84df-4463dd0d9686 kind: Event lastTimestamp: "2026-07-13T19:11:30Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-13T19:11:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:30Z" name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7.18c1ef29c48b37e5 namespace: kserve-ci-e2e-test resourceVersion: "20925" uid: a1c2d1a7-fff2-43a1-90c1-09a7900dbe50 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7 namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: c9b6e7e1-e413-42e0-84df-4463dd0d9686 kind: Event lastTimestamp: "2026-07-13T19:11:30Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-13T19:11:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:30Z" name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7.18c1ef29c4fbcfbb namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: d4d32e23-c0a1-4d3d-84a8-20ebe2965588 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7 namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: c9b6e7e1-e413-42e0-84df-4463dd0d9686 kind: Event lastTimestamp: "2026-07-13T19:11:34Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1730" already present on machine metadata: creationTimestamp: "2026-07-13T19:11:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:34Z" name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7.18c1ef2ab4e11d33 namespace: kserve-ci-e2e-test resourceVersion: "20959" uid: 64136c45-44d5-44a0-926a-8ef0ea048b47 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7 namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: c9b6e7e1-e413-42e0-84df-4463dd0d9686 kind: Event lastTimestamp: "2026-07-13T19:11:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:11:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:34Z" name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7.18c1ef2ab7ad0018 namespace: kserve-ci-e2e-test resourceVersion: "20961" uid: 787a0149-c365-47e7-84c1-4a691a1308a0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7 namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: c9b6e7e1-e413-42e0-84df-4463dd0d9686 kind: Event lastTimestamp: "2026-07-13T19:11:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:11:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:34Z" name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7.18c1ef2ab81c5ea1 namespace: kserve-ci-e2e-test resourceVersion: "20962" uid: 2fb59960-9c44-4d73-a21f-38792bfeba92 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7 namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: c9b6e7e1-e413-42e0-84df-4463dd0d9686 kind: Event lastTimestamp: "2026-07-13T19:11:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:11:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:34Z" name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7.18c1ef2ab825582c namespace: kserve-ci-e2e-test resourceVersion: "20963" uid: cc1feea9-0534-4178-aa69-3dbee6458adb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7 namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: c9b6e7e1-e413-42e0-84df-4463dd0d9686 kind: Event lastTimestamp: "2026-07-13T19:11:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:11:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:34Z" name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7.18c1ef2abb207700 namespace: kserve-ci-e2e-test resourceVersion: "20964" uid: 88d25ea4-f981-4e52-b612-a00454883e8f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7 namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: c9b6e7e1-e413-42e0-84df-4463dd0d9686 kind: Event lastTimestamp: "2026-07-13T19:11:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:11:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:34Z" name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7.18c1ef2abb8cd64f namespace: kserve-ci-e2e-test resourceVersion: "20965" uid: 7730a746-1036-4f40-b19d-4e277af9c9c6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-13T19:11:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7 namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: c9b6e7e1-e413-42e0-84df-4463dd0d9686 kind: Event lastTimestamp: "2026-07-13T19:13:02Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:11:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:02Z" name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7.18c1ef2af0e6f777 namespace: kserve-ci-e2e-test resourceVersion: "21621" uid: ac43138d-622e-4619-a2f0-91b31018ea13 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7 namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: c9b6e7e1-e413-42e0-84df-4463dd0d9686 kind: Event lastTimestamp: "2026-07-13T19:12:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:12:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:59Z" name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7.18c1ef3e995043f3 namespace: kserve-ci-e2e-test resourceVersion: "21598" uid: 4aae7ef1-24df-4a8d-b6b1-09ee54685a7e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7 namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: c9b6e7e1-e413-42e0-84df-4463dd0d9686 kind: Event lastTimestamp: "2026-07-13T19:12:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:12:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:59Z" name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7.18c1ef3e9950fb9e namespace: kserve-ci-e2e-test resourceVersion: "21599" uid: b4ed9108-e4a9-45df-9eb3-c96576aec4e1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7 namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: c9b6e7e1-e413-42e0-84df-4463dd0d9686 kind: Event lastTimestamp: "2026-07-13T19:13:02Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp 10.134.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:13:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:02Z" name: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7.18c1ef3f333a7d5e namespace: kserve-ci-e2e-test resourceVersion: "21620" uid: 16058479-6bc1-461c-abc5-0f333df7ccfc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-f1f2c-predictor-5c547d56ff namespace: kserve-ci-e2e-test resourceVersion: "20893" uid: f3eb054b-94c6-4235-8439-35d8c1c25e5e kind: Event lastTimestamp: "2026-07-13T19:11:29Z" message: 'Created pod: raw-sklearn-f1f2c-predictor-5c547d56ff-ffgs7' metadata: creationTimestamp: "2026-07-13T19:11:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:11:29Z" name: raw-sklearn-f1f2c-predictor-5c547d56ff.18c1ef29a5a9d7fd namespace: kserve-ci-e2e-test resourceVersion: "20910" uid: 645af78b-98d7-46ed-b40e-045d49df6f51 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-f1f2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20889" uid: 9ae492c4-587d-45a4-9035-62f3d2ffcfd2 kind: Event lastTimestamp: "2026-07-13T19:11:29Z" message: Scaled up replica set raw-sklearn-f1f2c-predictor-5c547d56ff from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:11:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:11:29Z" name: raw-sklearn-f1f2c-predictor.18c1ef299e332c32 namespace: kserve-ci-e2e-test resourceVersion: "20896" uid: 5946b1ec-c476-44c0-a7c5-f02c8f5637d7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:11:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-f1f2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20892" uid: 0ab8bf60-d2d6-47ab-b8d2-f52c34f6b1f9 kind: Event lastTimestamp: "2026-07-13T19:12:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:11:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:12:14Z" name: raw-sklearn-f1f2c-predictor.18c1ef2d1be6a33d namespace: kserve-ci-e2e-test resourceVersion: "21253" uid: c432b2ef-6754-425e-8678-ecf87e4fc36c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:11:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-f1f2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20892" uid: 0ab8bf60-d2d6-47ab-b8d2-f52c34f6b1f9 kind: Event lastTimestamp: "2026-07-13T19:12:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:11:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:12:14Z" name: raw-sklearn-f1f2c-predictor.18c1ef2d1c8e1293 namespace: kserve-ci-e2e-test resourceVersion: "21254" uid: 601e5b6e-8ef3-4300-ad7c-cf5919cff303 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:11:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-f1f2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21045" uid: 0ab8bf60-d2d6-47ab-b8d2-f52c34f6b1f9 kind: Event lastTimestamp: "2026-07-13T19:12:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:11:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:12:44Z" name: raw-sklearn-f1f2c-predictor.18c1ef309bf8a0b5 namespace: kserve-ci-e2e-test resourceVersion: "21465" uid: b9593ddb-0907-44d3-ba9e-93f9197da6a6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:11:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-f1f2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21045" uid: 0ab8bf60-d2d6-47ab-b8d2-f52c34f6b1f9 kind: Event lastTimestamp: "2026-07-13T19:12:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:11:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:12:44Z" name: raw-sklearn-f1f2c-predictor.18c1ef309ca648c9 namespace: kserve-ci-e2e-test resourceVersion: "21466" uid: f8421cf4-a185-476b-af86-ebede849a83f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-f1f2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21362" uid: 0ab8bf60-d2d6-47ab-b8d2-f52c34f6b1f9 kind: Event lastTimestamp: "2026-07-13T19:12:59Z" message: deployments.apps "raw-sklearn-f1f2c-predictor" not found metadata: creationTimestamp: "2026-07-13T19:12:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:12:59Z" name: raw-sklearn-f1f2c-predictor.18c1ef3e98853378 namespace: kserve-ci-e2e-test resourceVersion: "21593" uid: 6e35ed53-7fb0-4b9e-a3a7-062beafc6ad5 reason: FailedGetScale reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-f1f2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21362" uid: 0ab8bf60-d2d6-47ab-b8d2-f52c34f6b1f9 kind: Event lastTimestamp: "2026-07-13T19:12:59Z" message: horizontalpodautoscalers.autoscaling "raw-sklearn-f1f2c-predictor" not found metadata: creationTimestamp: "2026-07-13T19:12:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:12:59Z" name: raw-sklearn-f1f2c-predictor.18c1ef3e989e9e57 namespace: kserve-ci-e2e-test resourceVersion: "21596" uid: 098006b4-d1fb-4043-9984-6509ef646d58 reason: FailedUpdateStatus reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-13T19:11:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-f1f2c namespace: kserve-ci-e2e-test resourceVersion: "20884" uid: 230096a8-3202-47f8-b630-054fd714ddc1 kind: Event lastTimestamp: "2026-07-13T19:12:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:11:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:12:52Z" name: raw-sklearn-f1f2c.18c1ef299d5d5dba namespace: kserve-ci-e2e-test resourceVersion: "21530" uid: 41fe2cae-5a06-44c0-9d5f-9719567d5f76 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-f1f2c namespace: kserve-ci-e2e-test resourceVersion: "20884" uid: 230096a8-3202-47f8-b630-054fd714ddc1 kind: Event lastTimestamp: "2026-07-13T19:11:29Z" message: 'Failed to update status for InferenceService "raw-sklearn-f1f2c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-f1f2c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:11:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:11:29Z" name: raw-sklearn-f1f2c.18c1ef299e0b8b62 namespace: kserve-ci-e2e-test resourceVersion: "20897" uid: 879921be-98ec-43f9-b64e-1091bd3db0c4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-f1f2c namespace: kserve-ci-e2e-test resourceVersion: "21528" uid: 230096a8-3202-47f8-b630-054fd714ddc1 kind: Event lastTimestamp: "2026-07-13T19:12:52Z" message: InferenceService [raw-sklearn-f1f2c] is Ready metadata: creationTimestamp: "2026-07-13T19:12:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:12:52Z" name: raw-sklearn-f1f2c.18c1ef3ce4f6a9ab namespace: kserve-ci-e2e-test resourceVersion: "21529" uid: d14ee007-bf90-4421-9b1f-ad5f09e245b2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:13:04.729476Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865 namespace: kserve-ci-e2e-test resourceVersion: "21664" uid: c2d48667-72e8-463e-866a-ee00e4882c2a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865 to ip-10-0-130-96.ec2.internal metadata: creationTimestamp: "2026-07-13T19:13:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:13:04Z" name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865.18c1ef3fcdc28aeb namespace: kserve-ci-e2e-test resourceVersion: "21669" uid: 5f031a1f-b4ed-4a34-a526-e32083724a1d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-645b5db44b-6qs9d source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:05Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865 namespace: kserve-ci-e2e-test resourceVersion: "21673" uid: c2d48667-72e8-463e-866a-ee00e4882c2a kind: Event lastTimestamp: "2026-07-13T19:13:05Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:13:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:13:05Z" name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865.18c1ef3fe724ba5c namespace: kserve-ci-e2e-test resourceVersion: "21682" uid: 4b47affa-25fe-40f1-a212-5142c64b6f48 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865 namespace: kserve-ci-e2e-test resourceVersion: "21666" uid: c2d48667-72e8-463e-866a-ee00e4882c2a kind: Event lastTimestamp: "2026-07-13T19:13:05Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:38f70cb77701ee00560ecdde553dd84342a4d8f7cb0779aa5b508807d9895bf1" already present on machine metadata: creationTimestamp: "2026-07-13T19:13:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:05Z" name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865.18c1ef3fe8b03dca namespace: kserve-ci-e2e-test resourceVersion: "21685" uid: 5242d642-ac75-4cd0-ab1e-6ad5f368c1b1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865 namespace: kserve-ci-e2e-test resourceVersion: "21666" uid: c2d48667-72e8-463e-866a-ee00e4882c2a kind: Event lastTimestamp: "2026-07-13T19:13:05Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-13T19:13:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:05Z" name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865.18c1ef3feb725880 namespace: kserve-ci-e2e-test resourceVersion: "21686" uid: 948c85bf-8739-4edd-83ed-c988cf64bbea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865 namespace: kserve-ci-e2e-test resourceVersion: "21666" uid: c2d48667-72e8-463e-866a-ee00e4882c2a kind: Event lastTimestamp: "2026-07-13T19:13:05Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-13T19:13:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:05Z" name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865.18c1ef3febfd3436 namespace: kserve-ci-e2e-test resourceVersion: "21687" uid: 7eece31a-895e-4644-a214-87a1b85f693c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865 namespace: kserve-ci-e2e-test resourceVersion: "21666" uid: c2d48667-72e8-463e-866a-ee00e4882c2a kind: Event lastTimestamp: "2026-07-13T19:13:09Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1730" already present on machine metadata: creationTimestamp: "2026-07-13T19:13:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:09Z" name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865.18c1ef40e9891376 namespace: kserve-ci-e2e-test resourceVersion: "21722" uid: 60b467e4-0627-45ec-ba08-854659aaff9e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865 namespace: kserve-ci-e2e-test resourceVersion: "21666" uid: c2d48667-72e8-463e-866a-ee00e4882c2a kind: Event lastTimestamp: "2026-07-13T19:13:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:13:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:09Z" name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865.18c1ef40eca7ba1c namespace: kserve-ci-e2e-test resourceVersion: "21724" uid: e50ea110-edba-4a49-8cb0-f2c18cd0c1ae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865 namespace: kserve-ci-e2e-test resourceVersion: "21666" uid: c2d48667-72e8-463e-866a-ee00e4882c2a kind: Event lastTimestamp: "2026-07-13T19:13:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:13:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:09Z" name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865.18c1ef40ed378c54 namespace: kserve-ci-e2e-test resourceVersion: "21727" uid: cf3ffe89-6710-4654-9283-23a08afb3260 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865 namespace: kserve-ci-e2e-test resourceVersion: "21666" uid: c2d48667-72e8-463e-866a-ee00e4882c2a kind: Event lastTimestamp: "2026-07-13T19:13:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:13:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:09Z" name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865.18c1ef40ed4186eb namespace: kserve-ci-e2e-test resourceVersion: "21728" uid: 1151d2b1-a2c1-4774-a4fb-cb21f54d8df9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865 namespace: kserve-ci-e2e-test resourceVersion: "21666" uid: c2d48667-72e8-463e-866a-ee00e4882c2a kind: Event lastTimestamp: "2026-07-13T19:13:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:13:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:09Z" name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865.18c1ef40f05f4c37 namespace: kserve-ci-e2e-test resourceVersion: "21731" uid: 037f61cf-e8b0-42e1-aeb6-08e99a3678fc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865 namespace: kserve-ci-e2e-test resourceVersion: "21666" uid: c2d48667-72e8-463e-866a-ee00e4882c2a kind: Event lastTimestamp: "2026-07-13T19:13:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:13:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:09Z" name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865.18c1ef40f0ed7988 namespace: kserve-ci-e2e-test resourceVersion: "21732" uid: 2b5f074a-d60d-4211-87a7-56440640ac3b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-13T19:13:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865 namespace: kserve-ci-e2e-test resourceVersion: "21666" uid: c2d48667-72e8-463e-866a-ee00e4882c2a kind: Event lastTimestamp: "2026-07-13T19:14:37Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:14:37Z" name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865.18c1ef4125873684 namespace: kserve-ci-e2e-test resourceVersion: "22377" uid: 5ea5bc2d-a2b0-45e7-a264-f2873284d15a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:14:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865 namespace: kserve-ci-e2e-test resourceVersion: "21666" uid: c2d48667-72e8-463e-866a-ee00e4882c2a kind: Event lastTimestamp: "2026-07-13T19:14:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:14:34Z" name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865.18c1ef54ce1c55ad namespace: kserve-ci-e2e-test resourceVersion: "22360" uid: a333eaa1-aaf9-4979-bb84-b11e8a07c94e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:14:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865 namespace: kserve-ci-e2e-test resourceVersion: "21666" uid: c2d48667-72e8-463e-866a-ee00e4882c2a kind: Event lastTimestamp: "2026-07-13T19:14:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:14:34Z" name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865.18c1ef54ce1d0e4c namespace: kserve-ci-e2e-test resourceVersion: "22361" uid: 4a248e26-2154-4969-8228-ba5b0c32d10b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:14:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865 namespace: kserve-ci-e2e-test resourceVersion: "21666" uid: c2d48667-72e8-463e-866a-ee00e4882c2a kind: Event lastTimestamp: "2026-07-13T19:14:36Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:14:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:14:36Z" name: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865.18c1ef552bb8329e namespace: kserve-ci-e2e-test resourceVersion: "22374" uid: 2946b637-d5d5-4d2d-937a-1ac858e2ecd6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-96.ec2.internal source: component: kubelet host: ip-10-0-130-96.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-09af0-predictor-749f6b4655 namespace: kserve-ci-e2e-test resourceVersion: "21652" uid: 5cb7c8c3-81ed-461e-a860-6b66e5be3415 kind: Event lastTimestamp: "2026-07-13T19:13:04Z" message: 'Created pod: raw-sklearn-runtime-09af0-predictor-749f6b4655-5d865' metadata: creationTimestamp: "2026-07-13T19:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:13:04Z" name: raw-sklearn-runtime-09af0-predictor-749f6b4655.18c1ef3fcd327e5b namespace: kserve-ci-e2e-test resourceVersion: "21665" uid: 6a38465c-601d-4091-8230-616219049df9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-09af0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21650" uid: a3969cca-84ba-491e-b6c1-52a62a64d261 kind: Event lastTimestamp: "2026-07-13T19:13:04Z" message: Scaled up replica set raw-sklearn-runtime-09af0-predictor-749f6b4655 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:13:04Z" name: raw-sklearn-runtime-09af0-predictor.18c1ef3fc88adcda namespace: kserve-ci-e2e-test resourceVersion: "21654" uid: 9a035961-e108-467d-98ae-bea7da221431 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:13:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-09af0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21651" uid: b9663b5c-1dc7-48bc-bb7a-175aa3eb46ac kind: Event lastTimestamp: "2026-07-13T19:13:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:13:34Z" name: raw-sklearn-runtime-09af0-predictor.18c1ef434789d73c namespace: kserve-ci-e2e-test resourceVersion: "21912" uid: 17fad78b-a774-4e4a-888c-ecb6c4b54488 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:13:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-09af0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21651" uid: b9663b5c-1dc7-48bc-bb7a-175aa3eb46ac kind: Event lastTimestamp: "2026-07-13T19:13:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:13:34Z" name: raw-sklearn-runtime-09af0-predictor.18c1ef4348164d68 namespace: kserve-ci-e2e-test resourceVersion: "21913" uid: 055dd138-71fd-49d4-b9d7-f03f96a5ea0d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T19:13:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-09af0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21808" uid: b9663b5c-1dc7-48bc-bb7a-175aa3eb46ac kind: Event lastTimestamp: "2026-07-13T19:14:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:13:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:14:34Z" name: raw-sklearn-runtime-09af0-predictor.18c1ef4a46262092 namespace: kserve-ci-e2e-test resourceVersion: "22344" uid: b4415de4-ca32-426d-99be-3e76306900e5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T19:13:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-09af0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21808" uid: b9663b5c-1dc7-48bc-bb7a-175aa3eb46ac kind: Event lastTimestamp: "2026-07-13T19:14:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:13:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:14:34Z" name: raw-sklearn-runtime-09af0-predictor.18c1ef4a4687cd8f namespace: kserve-ci-e2e-test resourceVersion: "22345" uid: 05c169cc-57fe-47ed-ad35-05650d57a436 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-13T19:13:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-09af0 namespace: kserve-ci-e2e-test resourceVersion: "21643" uid: 9028f96b-8f90-40e0-bbfd-1df89ad5e8c2 kind: Event lastTimestamp: "2026-07-13T19:14:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:14:27Z" name: raw-sklearn-runtime-09af0.18c1ef3fc88e3653 namespace: kserve-ci-e2e-test resourceVersion: "22289" uid: 220077ed-2ac4-4d63-a50f-7dc4af8ced0f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-09af0 namespace: kserve-ci-e2e-test resourceVersion: "21643" uid: 9028f96b-8f90-40e0-bbfd-1df89ad5e8c2 kind: Event lastTimestamp: "2026-07-13T19:13:04Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-09af0": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-09af0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:13:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:13:04Z" name: raw-sklearn-runtime-09af0.18c1ef3fc95e101f namespace: kserve-ci-e2e-test resourceVersion: "21657" uid: 8433fcf0-1a68-42a4-8cd4-89ec82ba520b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:14:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-09af0 namespace: kserve-ci-e2e-test resourceVersion: "22287" uid: 9028f96b-8f90-40e0-bbfd-1df89ad5e8c2 kind: Event lastTimestamp: "2026-07-13T19:14:27Z" message: InferenceService [raw-sklearn-runtime-09af0] is Ready metadata: creationTimestamp: "2026-07-13T19:14:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:14:27Z" name: raw-sklearn-runtime-09af0.18c1ef52fd428381 namespace: kserve-ci-e2e-test resourceVersion: "22288" uid: c11c15ee-d3f5-419b-8740-cc1ffa60f22d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "22604"