--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:01:59.761499Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-f3d602-predictor-7bbdc69c6d-v895j namespace: kserve-ci-e2e-test resourceVersion: "21019" uid: 9faee4ca-523f-4a0c-80a4-1373ba260eaf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-f3d602-predictor-7bbdc69c6d-v895j to ip-10-0-143-88.ec2.internal metadata: creationTimestamp: "2026-07-02T13:01:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:01:59Z" name: isvc-init-fail-f3d602-predictor-7bbdc69c6d-v895j.18be7a9d97dbc509 namespace: kserve-ci-e2e-test resourceVersion: "21025" uid: a64254b4-0fe7-459e-9111-20f29349f5b8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-fcf8ff-ccrjf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:00Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-f3d602-predictor-7bbdc69c6d-v895j namespace: kserve-ci-e2e-test resourceVersion: "21029" uid: 9faee4ca-523f-4a0c-80a4-1373ba260eaf kind: Event lastTimestamp: "2026-07-02T13:02:00Z" message: Add eth0 [10.132.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:02:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:02:00Z" name: isvc-init-fail-f3d602-predictor-7bbdc69c6d-v895j.18be7a9db1a70185 namespace: kserve-ci-e2e-test resourceVersion: "21039" uid: 313e8d7d-04ff-41cc-a239-8af185768d82 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:02:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-f3d602-predictor-7bbdc69c6d-v895j namespace: kserve-ci-e2e-test resourceVersion: "21021" uid: 9faee4ca-523f-4a0c-80a4-1373ba260eaf kind: Event lastTimestamp: "2026-07-02T13:02:03Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ad817336055c6da6bf66cacbd06b857caf8adb146e60422365f2b38a3444569b" already present on machine metadata: creationTimestamp: "2026-07-02T13:02:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:03Z" name: isvc-init-fail-f3d602-predictor-7bbdc69c6d-v895j.18be7a9db3471b74 namespace: kserve-ci-e2e-test resourceVersion: "21077" uid: d73c9af4-c906-4f61-a3e8-87ee77ef180e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:02:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-f3d602-predictor-7bbdc69c6d-v895j namespace: kserve-ci-e2e-test resourceVersion: "21021" uid: 9faee4ca-523f-4a0c-80a4-1373ba260eaf kind: Event lastTimestamp: "2026-07-02T13:02:03Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T13:02:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:03Z" name: isvc-init-fail-f3d602-predictor-7bbdc69c6d-v895j.18be7a9db67dcc9a namespace: kserve-ci-e2e-test resourceVersion: "21079" uid: 0b96bceb-9a41-47a6-9de1-52af1a3d5ad0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:02:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-f3d602-predictor-7bbdc69c6d-v895j namespace: kserve-ci-e2e-test resourceVersion: "21021" uid: 9faee4ca-523f-4a0c-80a4-1373ba260eaf kind: Event lastTimestamp: "2026-07-02T13:02:03Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T13:02:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:03Z" name: isvc-init-fail-f3d602-predictor-7bbdc69c6d-v895j.18be7a9db6ef3f3d namespace: kserve-ci-e2e-test resourceVersion: "21080" uid: 08972c50-ee3a-4ee6-a18d-d4474c8c179d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:04Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-f3d602-predictor-7bbdc69c6d-v895j namespace: kserve-ci-e2e-test resourceVersion: "21021" uid: 9faee4ca-523f-4a0c-80a4-1373ba260eaf kind: Event lastTimestamp: "2026-07-02T13:02:04Z" message: Unable to retrieve some image pull secrets (fail-s3-sa-f3d602-dockercfg-nkg5p); attempting to pull the image may not succeed. metadata: creationTimestamp: "2026-07-02T13:02:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:04Z" name: isvc-init-fail-f3d602-predictor-7bbdc69c6d-v895j.18be7a9ec11da74f namespace: kserve-ci-e2e-test resourceVersion: "21095" uid: 638e80ab-b6cd-47c0-8ab3-fcabf4566b83 reason: FailedToRetrieveImagePullSecret reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:02:04Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-f3d602-predictor-7bbdc69c6d-v895j namespace: kserve-ci-e2e-test resourceVersion: "21021" uid: 9faee4ca-523f-4a0c-80a4-1373ba260eaf kind: Event lastTimestamp: "2026-07-02T13:02:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-init-fail-f3d602-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:02:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:06Z" name: isvc-init-fail-f3d602-predictor-7bbdc69c6d-v895j.18be7a9ec7658673 namespace: kserve-ci-e2e-test resourceVersion: "21124" uid: f471ac15-cd4a-4d85-b137-e3ee74408e47 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-f3d602-predictor-7bbdc69c6d-v895j namespace: kserve-ci-e2e-test resourceVersion: "21021" uid: 9faee4ca-523f-4a0c-80a4-1373ba260eaf kind: Event lastTimestamp: "2026-07-02T13:02:05Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-07-02T13:02:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:05Z" name: isvc-init-fail-f3d602-predictor-7bbdc69c6d-v895j.18be7a9efd00b00f namespace: kserve-ci-e2e-test resourceVersion: "21118" uid: 49801574-0e77-4d57-a465-07f739fade43 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-f3d602-predictor-7bbdc69c6d namespace: kserve-ci-e2e-test resourceVersion: "21010" uid: 05c67f18-1533-4dd5-811e-6b12b048fddf kind: Event lastTimestamp: "2026-07-02T13:01:59Z" message: 'Created pod: isvc-init-fail-f3d602-predictor-7bbdc69c6d-v895j' metadata: creationTimestamp: "2026-07-02T13:01:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:01:59Z" name: isvc-init-fail-f3d602-predictor-7bbdc69c6d.18be7a9d9770f38d namespace: kserve-ci-e2e-test resourceVersion: "21020" uid: 9f3a3f9d-f2af-4b10-b7f3-1037d7fe7567 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-f3d602-predictor namespace: kserve-ci-e2e-test resourceVersion: "21008" uid: 150192e8-3cac-4187-ac40-37eea32cfb7b kind: Event lastTimestamp: "2026-07-02T13:01:59Z" message: Scaled up replica set isvc-init-fail-f3d602-predictor-7bbdc69c6d from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:01:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:01:59Z" name: isvc-init-fail-f3d602-predictor.18be7a9d9465db4c namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: e12a631a-e24d-4bd3-9285-aedfdc9e933b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T13:01:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-f3d602 namespace: kserve-ci-e2e-test resourceVersion: "21000" uid: da14bd0c-2ba3-4e2f-b161-b9a44d346fa2 kind: Event lastTimestamp: "2026-07-02T13:02:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:01:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:02:03Z" name: isvc-init-fail-f3d602.18be7a9d946a6928 namespace: kserve-ci-e2e-test resourceVersion: "21083" uid: 9a1cf17f-f943-4013-9cc9-f3313c43d48e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-f3d602 namespace: kserve-ci-e2e-test resourceVersion: "21000" uid: da14bd0c-2ba3-4e2f-b161-b9a44d346fa2 kind: Event lastTimestamp: "2026-07-02T13:01:59Z" message: 'Failed to update status for InferenceService "isvc-init-fail-f3d602": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-f3d602": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:01:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:01:59Z" name: isvc-init-fail-f3d602.18be7a9d94d03874 namespace: kserve-ci-e2e-test resourceVersion: "21014" uid: 73d7eb1b-d95c-40ff-86d7-52a3c38dd85a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-f3d602 namespace: kserve-ci-e2e-test resourceVersion: "21000" uid: da14bd0c-2ba3-4e2f-b161-b9a44d346fa2 kind: Event lastTimestamp: "2026-07-02T13:01:59Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-f3d602": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:01:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:01:59Z" name: isvc-init-fail-f3d602.18be7a9d94d07c14 namespace: kserve-ci-e2e-test resourceVersion: "21015" uid: 65ea077e-ecd4-4d60-b5b5-428abe6af6c0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T12:55:45.655811Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l namespace: kserve-ci-e2e-test resourceVersion: "17975" uid: 6eb83fab-c7af-40e5-8059-88ff3b38cfcb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l to ip-10-0-143-88.ec2.internal metadata: creationTimestamp: "2026-07-02T12:55:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T12:55:45Z" name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l.18be7a467d6bf8b5 namespace: kserve-ci-e2e-test resourceVersion: "17980" uid: 62fdcfa5-cf8a-44d1-bb32-b4176e28a038 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-fcf8ff-ccrjf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:46Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l namespace: kserve-ci-e2e-test resourceVersion: "17983" uid: 6eb83fab-c7af-40e5-8059-88ff3b38cfcb kind: Event lastTimestamp: "2026-07-02T12:55:46Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T12:55:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T12:55:46Z" name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l.18be7a469767e8ad namespace: kserve-ci-e2e-test resourceVersion: "17995" uid: 5457bdfb-b6da-4b74-9011-393a4fa77070 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l namespace: kserve-ci-e2e-test resourceVersion: "17977" uid: 6eb83fab-c7af-40e5-8059-88ff3b38cfcb kind: Event lastTimestamp: "2026-07-02T12:55:46Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ad817336055c6da6bf66cacbd06b857caf8adb146e60422365f2b38a3444569b" already present on machine metadata: creationTimestamp: "2026-07-02T12:55:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:46Z" name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l.18be7a4699534b38 namespace: kserve-ci-e2e-test resourceVersion: "17997" uid: 560bdc43-fb1a-4a38-ba5d-5fb2a315779d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l namespace: kserve-ci-e2e-test resourceVersion: "17977" uid: 6eb83fab-c7af-40e5-8059-88ff3b38cfcb kind: Event lastTimestamp: "2026-07-02T12:55:46Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T12:55:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:46Z" name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l.18be7a469c153feb namespace: kserve-ci-e2e-test resourceVersion: "17998" uid: 589366fd-8d8d-4296-89bb-cad9250bf2cf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l namespace: kserve-ci-e2e-test resourceVersion: "17977" uid: 6eb83fab-c7af-40e5-8059-88ff3b38cfcb kind: Event lastTimestamp: "2026-07-02T12:55:46Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T12:55:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:46Z" name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l.18be7a469c88fb7e namespace: kserve-ci-e2e-test resourceVersion: "17999" uid: cdec6efa-d2f9-4339-ba00-69f9d255ae04 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l namespace: kserve-ci-e2e-test resourceVersion: "17977" uid: 6eb83fab-c7af-40e5-8059-88ff3b38cfcb kind: Event lastTimestamp: "2026-07-02T12:55:50Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1694" already present on machine metadata: creationTimestamp: "2026-07-02T12:55:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:50Z" name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l.18be7a47b4f368b2 namespace: kserve-ci-e2e-test resourceVersion: "18039" uid: 4cf607e6-d170-4e1a-ac89-558c26b6224a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l namespace: kserve-ci-e2e-test resourceVersion: "17977" uid: 6eb83fab-c7af-40e5-8059-88ff3b38cfcb kind: Event lastTimestamp: "2026-07-02T12:55:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T12:55:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:50Z" name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l.18be7a47b8322d29 namespace: kserve-ci-e2e-test resourceVersion: "18042" uid: a19b658d-69b7-4477-bd6f-67b0e6196c93 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l namespace: kserve-ci-e2e-test resourceVersion: "17977" uid: 6eb83fab-c7af-40e5-8059-88ff3b38cfcb kind: Event lastTimestamp: "2026-07-02T12:55:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T12:55:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:50Z" name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l.18be7a47b8a2c1ce namespace: kserve-ci-e2e-test resourceVersion: "18043" uid: 510b8f32-0891-45fc-973f-065d3ca18932 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l namespace: kserve-ci-e2e-test resourceVersion: "17977" uid: 6eb83fab-c7af-40e5-8059-88ff3b38cfcb kind: Event lastTimestamp: "2026-07-02T12:55:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T12:55:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:50Z" name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l.18be7a47b8acc818 namespace: kserve-ci-e2e-test resourceVersion: "18044" uid: 80a6248e-61ca-4f98-b829-89194772d6c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l namespace: kserve-ci-e2e-test resourceVersion: "17977" uid: 6eb83fab-c7af-40e5-8059-88ff3b38cfcb kind: Event lastTimestamp: "2026-07-02T12:55:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T12:55:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:51Z" name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l.18be7a47bc634bf1 namespace: kserve-ci-e2e-test resourceVersion: "18047" uid: fa7b43f7-04aa-45e2-b000-b0c2e67acfb7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l namespace: kserve-ci-e2e-test resourceVersion: "17977" uid: 6eb83fab-c7af-40e5-8059-88ff3b38cfcb kind: Event lastTimestamp: "2026-07-02T12:55:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:55:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:51Z" name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l.18be7a47bcd1f0e7 namespace: kserve-ci-e2e-test resourceVersion: "18048" uid: 04f41766-9b68-4b45-a19c-b8f7dc37335c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l namespace: kserve-ci-e2e-test resourceVersion: "17977" uid: 6eb83fab-c7af-40e5-8059-88ff3b38cfcb kind: Event lastTimestamp: "2026-07-02T12:55:51Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:4e7b5aaca6f327ecd44367d576730108bfee7dfd141d17da265242541f7cb6f0" already present on machine metadata: creationTimestamp: "2026-07-02T12:55:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:51Z" name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l.18be7a47bcdc85f9 namespace: kserve-ci-e2e-test resourceVersion: "18050" uid: 50c37627-9923-4cb3-8dee-34028c1f18ae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l namespace: kserve-ci-e2e-test resourceVersion: "17977" uid: 6eb83fab-c7af-40e5-8059-88ff3b38cfcb kind: Event lastTimestamp: "2026-07-02T12:55:51Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-02T12:55:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:51Z" name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l.18be7a47bfb23a29 namespace: kserve-ci-e2e-test resourceVersion: "18051" uid: 804e37d0-4d28-4a9e-9d30-2c59f76ad673 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l namespace: kserve-ci-e2e-test resourceVersion: "17977" uid: 6eb83fab-c7af-40e5-8059-88ff3b38cfcb kind: Event lastTimestamp: "2026-07-02T12:55:51Z" message: Started container agent metadata: creationTimestamp: "2026-07-02T12:55:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:51Z" name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l.18be7a47c023549a namespace: kserve-ci-e2e-test resourceVersion: "18052" uid: 364e8d60-2e54-4113-9408-9969220390ce reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T12:55:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l namespace: kserve-ci-e2e-test resourceVersion: "17977" uid: 6eb83fab-c7af-40e5-8059-88ff3b38cfcb kind: Event lastTimestamp: "2026-07-02T12:57:27Z" message: 'Readiness probe failed: dial tcp 10.132.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T12:55:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:57:27Z" name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l.18be7a47f110b1ec namespace: kserve-ci-e2e-test resourceVersion: "18864" uid: b5867d3f-1fa4-4df6-a808-be93b397f178 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T12:55:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l namespace: kserve-ci-e2e-test resourceVersion: "17977" uid: 6eb83fab-c7af-40e5-8059-88ff3b38cfcb kind: Event lastTimestamp: "2026-07-02T12:57:37Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T12:55:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:57:37Z" name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l.18be7a47f11c0416 namespace: kserve-ci-e2e-test resourceVersion: "18941" uid: 2136b93d-2043-4a08-b119-2de77d52738d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:57:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l namespace: kserve-ci-e2e-test resourceVersion: "17977" uid: 6eb83fab-c7af-40e5-8059-88ff3b38cfcb kind: Event lastTimestamp: "2026-07-02T12:57:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T12:57:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:57:20Z" name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l.18be7a5cadcb030c namespace: kserve-ci-e2e-test resourceVersion: "18761" uid: 54c2f680-b1e2-4e5d-8e67-c81ca2834199 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:57:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l namespace: kserve-ci-e2e-test resourceVersion: "17977" uid: 6eb83fab-c7af-40e5-8059-88ff3b38cfcb kind: Event lastTimestamp: "2026-07-02T12:57:20Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-02T12:57:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:57:20Z" name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l.18be7a5cadcb3ecd namespace: kserve-ci-e2e-test resourceVersion: "18767" uid: fc91d945-8b3d-497e-8fb7-b53a4d08bb0b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:57:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l namespace: kserve-ci-e2e-test resourceVersion: "17977" uid: 6eb83fab-c7af-40e5-8059-88ff3b38cfcb kind: Event lastTimestamp: "2026-07-02T12:57:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:57:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:57:20Z" name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l.18be7a5cadcb55f1 namespace: kserve-ci-e2e-test resourceVersion: "18773" uid: 6ebc809e-31b2-42bd-91d3-3032e38a188d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T12:57:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l namespace: kserve-ci-e2e-test resourceVersion: "17977" uid: 6eb83fab-c7af-40e5-8059-88ff3b38cfcb kind: Event lastTimestamp: "2026-07-02T12:57:37Z" message: 'Readiness probe failed: Get "https://10.132.0.42:8643/healthz": dial tcp 10.132.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T12:57:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:57:37Z" name: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l.18be7a5d21552886 namespace: kserve-ci-e2e-test resourceVersion: "18940" uid: bc5f8146-4f31-4280-918c-a47dc92124b4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-c99c3-predictor-77cbd7b758 namespace: kserve-ci-e2e-test resourceVersion: "17964" uid: 9692fd1f-7a41-4cc6-92b3-89cf4e6c090b kind: Event lastTimestamp: "2026-07-02T12:55:45Z" message: 'Created pod: isvc-logger-raw-c99c3-predictor-77cbd7b758-qd74l' metadata: creationTimestamp: "2026-07-02T12:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:55:45Z" name: isvc-logger-raw-c99c3-predictor-77cbd7b758.18be7a467ceacec4 namespace: kserve-ci-e2e-test resourceVersion: "17976" uid: 8811983c-3dcb-4417-84df-88df49659fef reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-c99c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "17962" uid: 52b90368-bb1f-4439-9fad-4bc940b59e57 kind: Event lastTimestamp: "2026-07-02T12:55:45Z" message: Scaled up replica set isvc-logger-raw-c99c3-predictor-77cbd7b758 from 0 to 1 metadata: creationTimestamp: "2026-07-02T12:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:55:45Z" name: isvc-logger-raw-c99c3-predictor.18be7a4677ad66da namespace: kserve-ci-e2e-test resourceVersion: "17965" uid: 4a41fe05-246a-4c74-833f-3e2d30ff2ec3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:56:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-c99c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "17963" uid: 2b284ee2-5260-4a47-a571-5e241657b74b kind: Event lastTimestamp: "2026-07-02T12:56:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:56:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:56:30Z" name: isvc-logger-raw-c99c3-predictor.18be7a49f629ec8c namespace: kserve-ci-e2e-test resourceVersion: "18347" uid: 45cb282e-8446-411e-a310-5f2a8ccaae77 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:56:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-c99c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "17963" uid: 2b284ee2-5260-4a47-a571-5e241657b74b kind: Event lastTimestamp: "2026-07-02T12:56:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:56:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:56:30Z" name: isvc-logger-raw-c99c3-predictor.18be7a49f691f47f namespace: kserve-ci-e2e-test resourceVersion: "18348" uid: 431584b7-0c16-479c-8e27-ae86c1d16dbf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:56:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-c99c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "18138" uid: 2b284ee2-5260-4a47-a571-5e241657b74b kind: Event lastTimestamp: "2026-07-02T12:57:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:57:15Z" name: isvc-logger-raw-c99c3-predictor.18be7a5472d986f9 namespace: kserve-ci-e2e-test resourceVersion: "18674" uid: daea64f6-4366-461d-86f4-761fa746c22e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:56:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-c99c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "18138" uid: 2b284ee2-5260-4a47-a571-5e241657b74b kind: Event lastTimestamp: "2026-07-02T12:57:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:57:15Z" name: isvc-logger-raw-c99c3-predictor.18be7a54734043c9 namespace: kserve-ci-e2e-test resourceVersion: "18675" uid: 70354d75-245d-49c6-930b-4ce47e175914 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T12:55:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-c99c3 namespace: kserve-ci-e2e-test resourceVersion: "17955" uid: 78704da2-43a2-42a1-b087-09a875edb481 kind: Event lastTimestamp: "2026-07-02T12:57:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T12:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:57:08Z" name: isvc-logger-raw-c99c3.18be7a4677e1a7c7 namespace: kserve-ci-e2e-test resourceVersion: "18621" uid: 71922b7f-f0c5-43c4-941a-47863e100bd4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-c99c3 namespace: kserve-ci-e2e-test resourceVersion: "17955" uid: 78704da2-43a2-42a1-b087-09a875edb481 kind: Event lastTimestamp: "2026-07-02T12:55:45Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-c99c3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-c99c3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T12:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:55:45Z" name: isvc-logger-raw-c99c3.18be7a46785c5679 namespace: kserve-ci-e2e-test resourceVersion: "17968" uid: 2472d067-589c-47f6-950a-e00fcc4f24ea reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:57:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-c99c3 namespace: kserve-ci-e2e-test resourceVersion: "18618" uid: 78704da2-43a2-42a1-b087-09a875edb481 kind: Event lastTimestamp: "2026-07-02T12:57:08Z" message: InferenceService [isvc-logger-raw-c99c3] is Ready metadata: creationTimestamp: "2026-07-02T12:57:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:57:08Z" name: isvc-logger-raw-c99c3.18be7a59aa6fb8ca namespace: kserve-ci-e2e-test resourceVersion: "18619" uid: 5a8c4231-c92b-4ff4-bad0-b9709d54d604 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:00:06.241157Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 80847d7c-5d90-449c-816c-70cdb1548320 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt to ip-10-0-143-88.ec2.internal metadata: creationTimestamp: "2026-07-02T13:00:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:00:06Z" name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt.18be7a832984ca50 namespace: kserve-ci-e2e-test resourceVersion: "20035" uid: ede74d8b-f6e4-47a2-8d9a-a30c39c30a5e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-fcf8ff-ccrjf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:06Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 80847d7c-5d90-449c-816c-70cdb1548320 kind: Event lastTimestamp: "2026-07-02T13:00:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-primary-c9e60f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:00:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:06Z" name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt.18be7a8332f5dbb0 namespace: kserve-ci-e2e-test resourceVersion: "20045" uid: c69f1a67-d4f8-4065-a22e-750ac369a700 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:07Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt namespace: kserve-ci-e2e-test resourceVersion: "20039" uid: 80847d7c-5d90-449c-816c-70cdb1548320 kind: Event lastTimestamp: "2026-07-02T13:00:07Z" message: Add eth0 [10.132.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:00:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:00:07Z" name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt.18be7a83671f0788 namespace: kserve-ci-e2e-test resourceVersion: "20057" uid: b445c6cf-4ea4-4ebe-bfad-fb3dc8470ac1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 80847d7c-5d90-449c-816c-70cdb1548320 kind: Event lastTimestamp: "2026-07-02T13:00:07Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ad817336055c6da6bf66cacbd06b857caf8adb146e60422365f2b38a3444569b" already present on machine metadata: creationTimestamp: "2026-07-02T13:00:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:07Z" name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt.18be7a8369101854 namespace: kserve-ci-e2e-test resourceVersion: "20059" uid: 6e667207-f93c-4256-94fe-599f495843aa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 80847d7c-5d90-449c-816c-70cdb1548320 kind: Event lastTimestamp: "2026-07-02T13:00:07Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T13:00:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:07Z" name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt.18be7a836bdacd30 namespace: kserve-ci-e2e-test resourceVersion: "20060" uid: efb405e2-762e-473a-9299-5f8c2af0c50d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 80847d7c-5d90-449c-816c-70cdb1548320 kind: Event lastTimestamp: "2026-07-02T13:00:07Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T13:00:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:07Z" name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt.18be7a836c6bfe9c namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 0884fe3c-8e35-4401-9e97-6aa8a3529dc0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 80847d7c-5d90-449c-816c-70cdb1548320 kind: Event lastTimestamp: "2026-07-02T13:00:11Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1694" already present on machine metadata: creationTimestamp: "2026-07-02T13:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:11Z" name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt.18be7a845015fba3 namespace: kserve-ci-e2e-test resourceVersion: "20094" uid: 79ed48ea-4dc3-4812-8857-e86515f95dad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 80847d7c-5d90-449c-816c-70cdb1548320 kind: Event lastTimestamp: "2026-07-02T13:00:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:11Z" name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt.18be7a84538302e4 namespace: kserve-ci-e2e-test resourceVersion: "20098" uid: 984b313d-8c3b-4125-9681-b49f5b48f03e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 80847d7c-5d90-449c-816c-70cdb1548320 kind: Event lastTimestamp: "2026-07-02T13:00:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:11Z" name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt.18be7a8453f88128 namespace: kserve-ci-e2e-test resourceVersion: "20099" uid: 5f33dd2e-2550-40b1-a120-f14d84ebc766 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 80847d7c-5d90-449c-816c-70cdb1548320 kind: Event lastTimestamp: "2026-07-02T13:00:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:11Z" name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt.18be7a845401d094 namespace: kserve-ci-e2e-test resourceVersion: "20101" uid: b2887a0d-45a3-4925-87c0-87e5e4b07eef reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 80847d7c-5d90-449c-816c-70cdb1548320 kind: Event lastTimestamp: "2026-07-02T13:00:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:11Z" name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt.18be7a84570ef38c namespace: kserve-ci-e2e-test resourceVersion: "20104" uid: 7b0dc5a2-069b-4eea-acf0-c689778352f3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 80847d7c-5d90-449c-816c-70cdb1548320 kind: Event lastTimestamp: "2026-07-02T13:00:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:11Z" name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt.18be7a84577cfe86 namespace: kserve-ci-e2e-test resourceVersion: "20105" uid: 6f79d173-350d-41b9-8c65-5477238ca573 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T13:00:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 80847d7c-5d90-449c-816c-70cdb1548320 kind: Event lastTimestamp: "2026-07-02T13:01:58Z" message: 'Readiness probe failed: dial tcp 10.132.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:00:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:58Z" name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt.18be7a848c45a1f3 namespace: kserve-ci-e2e-test resourceVersion: "20980" uid: 71ffa0d1-24c7-4b57-9c37-4c892b9d844c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 80847d7c-5d90-449c-816c-70cdb1548320 kind: Event lastTimestamp: "2026-07-02T13:01:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:01:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:54Z" name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt.18be7a9c6793913e namespace: kserve-ci-e2e-test resourceVersion: "20946" uid: 1a0ac0ee-5bcf-4417-a8dc-659c4c2ed62c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 80847d7c-5d90-449c-816c-70cdb1548320 kind: Event lastTimestamp: "2026-07-02T13:01:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:01:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:54Z" name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt.18be7a9c679408a2 namespace: kserve-ci-e2e-test resourceVersion: "20947" uid: 9c84de72-59bb-4c40-9a82-d2e23c3155e9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 80847d7c-5d90-449c-816c-70cdb1548320 kind: Event lastTimestamp: "2026-07-02T13:01:58Z" message: 'Readiness probe failed: Get "https://10.132.0.44:8643/healthz": dial tcp 10.132.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:58Z" name: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt.18be7a9d3aa1db6f namespace: kserve-ci-e2e-test resourceVersion: "20979" uid: 0a457ab1-067d-4844-abde-326a96f1cfb8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-c9e60f-predictor-6d4687f95f namespace: kserve-ci-e2e-test resourceVersion: "20021" uid: 20eac526-ae34-4f5b-bdac-c83e20f868be kind: Event lastTimestamp: "2026-07-02T13:00:06Z" message: 'Created pod: isvc-primary-c9e60f-predictor-6d4687f95f-6qmdt' metadata: creationTimestamp: "2026-07-02T13:00:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:00:06Z" name: isvc-primary-c9e60f-predictor-6d4687f95f.18be7a83290e2c91 namespace: kserve-ci-e2e-test resourceVersion: "20032" uid: c5b49ef0-5d61-4301-8e22-771d750feb1e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-c9e60f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20020" uid: 43e1fac2-7c5f-4770-a115-38e9c18f69e7 kind: Event lastTimestamp: "2026-07-02T13:00:06Z" message: Scaled up replica set isvc-primary-c9e60f-predictor-6d4687f95f from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:00:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:00:06Z" name: isvc-primary-c9e60f-predictor.18be7a83236194df namespace: kserve-ci-e2e-test resourceVersion: "20023" uid: 24bad3f2-032e-41d4-ba8b-4afc05a0ca25 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-c9e60f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20022" uid: cf2f859e-626a-4131-a2d7-cd5787907102 kind: Event lastTimestamp: "2026-07-02T13:00:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:00:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:00:21Z" name: isvc-primary-c9e60f-predictor.18be7a86a2827be6 namespace: kserve-ci-e2e-test resourceVersion: "20183" uid: 011677da-900b-43cf-9523-5682b71e786b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-c9e60f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20022" uid: cf2f859e-626a-4131-a2d7-cd5787907102 kind: Event lastTimestamp: "2026-07-02T13:00:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:00:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:00:21Z" name: isvc-primary-c9e60f-predictor.18be7a86a2edf5c9 namespace: kserve-ci-e2e-test resourceVersion: "20185" uid: b4d883b8-38c8-42b1-85ef-2818a69d3154 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:00:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-c9e60f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20184" uid: cf2f859e-626a-4131-a2d7-cd5787907102 kind: Event lastTimestamp: "2026-07-02T13:01:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:01:51Z" name: isvc-primary-c9e60f-predictor.18be7a8a222273ba namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 2b4ae936-c99b-4014-9c00-b58144c9c82c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:00:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-c9e60f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20184" uid: cf2f859e-626a-4131-a2d7-cd5787907102 kind: Event lastTimestamp: "2026-07-02T13:01:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:01:51Z" name: isvc-primary-c9e60f-predictor.18be7a8a22aae4b6 namespace: kserve-ci-e2e-test resourceVersion: "20891" uid: 51c465b8-9572-46ec-ad30-7b164a756211 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T13:00:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-c9e60f namespace: kserve-ci-e2e-test resourceVersion: "20013" uid: 4ba8af67-c8d6-4191-88f9-d695db188981 kind: Event lastTimestamp: "2026-07-02T13:01:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:00:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:01:28Z" name: isvc-primary-c9e60f.18be7a8323f7f754 namespace: kserve-ci-e2e-test resourceVersion: "20663" uid: 06de2868-5dcc-4898-8358-2892b2d30a41 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:00:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-c9e60f namespace: kserve-ci-e2e-test resourceVersion: "20013" uid: 4ba8af67-c8d6-4191-88f9-d695db188981 kind: Event lastTimestamp: "2026-07-02T13:00:06Z" message: 'Failed to update status for InferenceService "isvc-primary-c9e60f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-c9e60f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:00:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:00:06Z" name: isvc-primary-c9e60f.18be7a83246fd1ae namespace: kserve-ci-e2e-test resourceVersion: "20041" uid: 05032dda-a492-4103-9394-05d01e01afcb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:00:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-c9e60f namespace: kserve-ci-e2e-test resourceVersion: "20013" uid: 4ba8af67-c8d6-4191-88f9-d695db188981 kind: Event lastTimestamp: "2026-07-02T13:00:06Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-c9e60f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:00:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:00:06Z" name: isvc-primary-c9e60f.18be7a8324701b5f namespace: kserve-ci-e2e-test resourceVersion: "20042" uid: 9c07f3e5-0807-46ea-96ad-b60a3e90970d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-c9e60f namespace: kserve-ci-e2e-test resourceVersion: "20660" uid: 4ba8af67-c8d6-4191-88f9-d695db188981 kind: Event lastTimestamp: "2026-07-02T13:01:28Z" message: InferenceService [isvc-primary-c9e60f] is Ready metadata: creationTimestamp: "2026-07-02T13:01:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:01:28Z" name: isvc-primary-c9e60f.18be7a9645ff497a namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: 538befb6-093c-4fbf-82af-59c91a05696b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T12:49:39.714138Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz namespace: kserve-ci-e2e-test resourceVersion: "14684" uid: fb23c80e-8cef-4219-9efd-ac6ed2cc8196 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz to ip-10-0-143-88.ec2.internal metadata: creationTimestamp: "2026-07-02T12:49:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T12:49:39Z" name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz.18be79f149997829 namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: 700fb066-cd0e-49fb-be83-be426071100e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-fcf8ff-ccrjf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:49:40Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz namespace: kserve-ci-e2e-test resourceVersion: "14694" uid: fb23c80e-8cef-4219-9efd-ac6ed2cc8196 kind: Event lastTimestamp: "2026-07-02T12:49:40Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T12:49:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T12:49:40Z" name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz.18be79f163a6d105 namespace: kserve-ci-e2e-test resourceVersion: "14700" uid: 0d20ebc8-b61d-4cbb-a6f6-01bb118925cd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:49:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: fb23c80e-8cef-4219-9efd-ac6ed2cc8196 kind: Event lastTimestamp: "2026-07-02T12:49:40Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ad817336055c6da6bf66cacbd06b857caf8adb146e60422365f2b38a3444569b" already present on machine metadata: creationTimestamp: "2026-07-02T12:49:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:49:40Z" name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz.18be79f1652de9aa namespace: kserve-ci-e2e-test resourceVersion: "14702" uid: f5d10903-f508-4b76-a990-8aef46f143cc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:49:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: fb23c80e-8cef-4219-9efd-ac6ed2cc8196 kind: Event lastTimestamp: "2026-07-02T12:49:40Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T12:49:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:49:40Z" name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz.18be79f168791693 namespace: kserve-ci-e2e-test resourceVersion: "14703" uid: 45b6dd7f-cd3d-453e-8b94-cd02f6e9e762 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:49:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: fb23c80e-8cef-4219-9efd-ac6ed2cc8196 kind: Event lastTimestamp: "2026-07-02T12:49:40Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T12:49:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:49:40Z" name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz.18be79f169076721 namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 8f22090d-0c13-4987-b808-de7e0f491818 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:49:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: fb23c80e-8cef-4219-9efd-ac6ed2cc8196 kind: Event lastTimestamp: "2026-07-02T12:49:45Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1694" metadata: creationTimestamp: "2026-07-02T12:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:49:45Z" name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz.18be79f2854b6182 namespace: kserve-ci-e2e-test resourceVersion: "14741" uid: 03a31bef-ac7e-481e-97b1-84daa4f60f54 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:49:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: fb23c80e-8cef-4219-9efd-ac6ed2cc8196 kind: Event lastTimestamp: "2026-07-02T12:49:58Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1694" in 13.912s (13.912s including waiting). Image size: 1558018474 bytes.' metadata: creationTimestamp: "2026-07-02T12:49:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:49:58Z" name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz.18be79f5c2894bed namespace: kserve-ci-e2e-test resourceVersion: "14851" uid: bf603702-3bb6-4f30-8078-27dfa5d229d3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:49:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: fb23c80e-8cef-4219-9efd-ac6ed2cc8196 kind: Event lastTimestamp: "2026-07-02T12:49:58Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T12:49:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:49:58Z" name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz.18be79f5c5c58dad namespace: kserve-ci-e2e-test resourceVersion: "14852" uid: 4804001a-0427-465c-962f-def10048e5f8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:49:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: fb23c80e-8cef-4219-9efd-ac6ed2cc8196 kind: Event lastTimestamp: "2026-07-02T12:49:58Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T12:49:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:49:58Z" name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz.18be79f5c636ac84 namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: 2af9735a-f134-4b89-bc7c-0542e2b8375f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:49:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: fb23c80e-8cef-4219-9efd-ac6ed2cc8196 kind: Event lastTimestamp: "2026-07-02T12:49:58Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T12:49:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:49:58Z" name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz.18be79f5c6444c51 namespace: kserve-ci-e2e-test resourceVersion: "14854" uid: b3bc09b1-c1fa-4015-b788-cbf0c98eca99 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: fb23c80e-8cef-4219-9efd-ac6ed2cc8196 kind: Event lastTimestamp: "2026-07-02T12:50:01Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.212s (2.212s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T12:50:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:01Z" name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz.18be79f64a22db7e namespace: kserve-ci-e2e-test resourceVersion: "14873" uid: f28fcfd2-d109-478f-ac47-605ff8c26e8b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: fb23c80e-8cef-4219-9efd-ac6ed2cc8196 kind: Event lastTimestamp: "2026-07-02T12:50:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T12:50:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:01Z" name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz.18be79f64d8b9633 namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: efe24c28-ca3f-4180-8b78-e1c25d021436 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: fb23c80e-8cef-4219-9efd-ac6ed2cc8196 kind: Event lastTimestamp: "2026-07-02T12:50:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:50:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:01Z" name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz.18be79f64dffd78b namespace: kserve-ci-e2e-test resourceVersion: "14877" uid: ad291dc9-265d-4ed7-aced-2b7c6a537ef4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: fb23c80e-8cef-4219-9efd-ac6ed2cc8196 kind: Event lastTimestamp: "2026-07-02T12:50:01Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:4e7b5aaca6f327ecd44367d576730108bfee7dfd141d17da265242541f7cb6f0" metadata: creationTimestamp: "2026-07-02T12:50:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:01Z" name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz.18be79f64e0df462 namespace: kserve-ci-e2e-test resourceVersion: "14878" uid: c5175a02-40e1-4423-a8c2-d4e7d71e3344 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: fb23c80e-8cef-4219-9efd-ac6ed2cc8196 kind: Event lastTimestamp: "2026-07-02T12:50:04Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:4e7b5aaca6f327ecd44367d576730108bfee7dfd141d17da265242541f7cb6f0" in 3.274s (3.274s including waiting). Image size: 239994468 bytes.' metadata: creationTimestamp: "2026-07-02T12:50:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:04Z" name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz.18be79f711365ef2 namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 0e567f00-f42b-4ebd-8c7d-1d6f4958662a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: fb23c80e-8cef-4219-9efd-ac6ed2cc8196 kind: Event lastTimestamp: "2026-07-02T12:50:04Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-02T12:50:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:04Z" name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz.18be79f7144be575 namespace: kserve-ci-e2e-test resourceVersion: "14897" uid: 3eab9a85-d838-45f2-80bd-2a80a1dc3930 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: fb23c80e-8cef-4219-9efd-ac6ed2cc8196 kind: Event lastTimestamp: "2026-07-02T12:50:04Z" message: Started container agent metadata: creationTimestamp: "2026-07-02T12:50:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:04Z" name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz.18be79f7151c375a namespace: kserve-ci-e2e-test resourceVersion: "14899" uid: 4d28cea3-c73e-497a-b027-d971c184a7ea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T12:50:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: fb23c80e-8cef-4219-9efd-ac6ed2cc8196 kind: Event lastTimestamp: "2026-07-02T12:51:39Z" message: 'Readiness probe failed: dial tcp 10.132.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T12:50:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:39Z" name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz.18be79f7354791d7 namespace: kserve-ci-e2e-test resourceVersion: "15722" uid: a093d8d1-200a-48e1-a1b6-aaad2b046c4c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T12:50:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: fb23c80e-8cef-4219-9efd-ac6ed2cc8196 kind: Event lastTimestamp: "2026-07-02T12:51:39Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T12:50:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:39Z" name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz.18be79f7712099b2 namespace: kserve-ci-e2e-test resourceVersion: "15723" uid: 45172be5-b90d-430b-9b6b-36bba2910f6d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: fb23c80e-8cef-4219-9efd-ac6ed2cc8196 kind: Event lastTimestamp: "2026-07-02T12:51:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T12:51:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:34Z" name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz.18be7a0c16094b8b namespace: kserve-ci-e2e-test resourceVersion: "15596" uid: f62fe5c1-58a4-4960-b222-07052de7c26a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: fb23c80e-8cef-4219-9efd-ac6ed2cc8196 kind: Event lastTimestamp: "2026-07-02T12:51:34Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-02T12:51:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:34Z" name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz.18be7a0c16099ba7 namespace: kserve-ci-e2e-test resourceVersion: "15600" uid: cb865bf8-3e02-46f5-86ef-08c6a523247c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: fb23c80e-8cef-4219-9efd-ac6ed2cc8196 kind: Event lastTimestamp: "2026-07-02T12:51:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:51:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:34Z" name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz.18be7a0c1609b630 namespace: kserve-ci-e2e-test resourceVersion: "15606" uid: 7f95f874-6f8e-4aee-b154-28b75d95c9d4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T12:51:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: fb23c80e-8cef-4219-9efd-ac6ed2cc8196 kind: Event lastTimestamp: "2026-07-02T12:51:41Z" message: 'Readiness probe failed: Get "https://10.132.0.37:8643/healthz": dial tcp 10.132.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T12:51:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:41Z" name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz.18be7a0c658298cd namespace: kserve-ci-e2e-test resourceVersion: "15736" uid: 26f68fcf-63f6-4876-9e58-95aa8db67da0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:49:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 1c3af0a0-5186-4166-846e-033b28ebda0b kind: Event lastTimestamp: "2026-07-02T12:49:39Z" message: 'Created pod: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd-twccz' metadata: creationTimestamp: "2026-07-02T12:49:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:49:39Z" name: isvc-raw-sklearn-batcher-2c740-predictor-65574885fd.18be79f1492f0c01 namespace: kserve-ci-e2e-test resourceVersion: "14685" uid: 674d6c0c-c512-4a7e-92d3-9650be94f8fe reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:49:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-2c740-predictor namespace: kserve-ci-e2e-test resourceVersion: "14668" uid: 6f73f5ac-26be-4beb-9791-5bb3e930a458 kind: Event lastTimestamp: "2026-07-02T12:49:39Z" message: Scaled up replica set isvc-raw-sklearn-batcher-2c740-predictor-65574885fd from 0 to 1 metadata: creationTimestamp: "2026-07-02T12:49:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:49:39Z" name: isvc-raw-sklearn-batcher-2c740-predictor.18be79f1401cd9b5 namespace: kserve-ci-e2e-test resourceVersion: "14670" uid: 8fdaf954-fb01-473d-ab54-c06e97e0970d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:49:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-2c740-predictor namespace: kserve-ci-e2e-test resourceVersion: "14679" uid: 6f4c8f9b-693c-4b0b-a658-eed5da0b1abb kind: Event lastTimestamp: "2026-07-02T12:50:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:49:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:50:24Z" name: isvc-raw-sklearn-batcher-2c740-predictor.18be79f4c536e557 namespace: kserve-ci-e2e-test resourceVersion: "15052" uid: a39e7599-ee8d-4b18-900d-baff67ae58ee reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:49:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-2c740-predictor namespace: kserve-ci-e2e-test resourceVersion: "14679" uid: 6f4c8f9b-693c-4b0b-a658-eed5da0b1abb kind: Event lastTimestamp: "2026-07-02T12:50:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:49:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:50:24Z" name: isvc-raw-sklearn-batcher-2c740-predictor.18be79f4c59f2abe namespace: kserve-ci-e2e-test resourceVersion: "15053" uid: bde2df67-91c9-4131-9419-0613d7b2d5f4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T12:50:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-2c740-predictor namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: 6f4c8f9b-693c-4b0b-a658-eed5da0b1abb kind: Event lastTimestamp: "2026-07-02T12:51:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:50:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:51:24Z" name: isvc-raw-sklearn-batcher-2c740-predictor.18be79ff3f30f4a1 namespace: kserve-ci-e2e-test resourceVersion: "15486" uid: ae6f07c8-222f-4b61-89ed-e0f258612e9a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T12:50:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-2c740-predictor namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: 6f4c8f9b-693c-4b0b-a658-eed5da0b1abb kind: Event lastTimestamp: "2026-07-02T12:51:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:50:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:51:24Z" name: isvc-raw-sklearn-batcher-2c740-predictor.18be79ff3f9673e7 namespace: kserve-ci-e2e-test resourceVersion: "15487" uid: 7efe3e4d-b48b-45ca-8dc1-df546ae64b49 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T12:49:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-2c740 namespace: kserve-ci-e2e-test resourceVersion: "14660" uid: 14ac2015-7d99-4b89-8baf-2bdec5f85aa2 kind: Event lastTimestamp: "2026-07-02T12:51:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T12:49:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:51:19Z" name: isvc-raw-sklearn-batcher-2c740.18be79f14636841a namespace: kserve-ci-e2e-test resourceVersion: "15454" uid: 4d276344-e9d7-4bd7-97ae-b5bb9a7fbafb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:49:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-2c740 namespace: kserve-ci-e2e-test resourceVersion: "14660" uid: 14ac2015-7d99-4b89-8baf-2bdec5f85aa2 kind: Event lastTimestamp: "2026-07-02T12:49:39Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-2c740": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-2c740": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T12:49:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:49:39Z" name: isvc-raw-sklearn-batcher-2c740.18be79f1468478da namespace: kserve-ci-e2e-test resourceVersion: "14682" uid: 9f3adaf0-0bdb-4dca-ab9b-ad50490e6dde reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:49:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-2c740 namespace: kserve-ci-e2e-test resourceVersion: "14660" uid: 14ac2015-7d99-4b89-8baf-2bdec5f85aa2 kind: Event lastTimestamp: "2026-07-02T12:49:39Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-2c740": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T12:49:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:49:39Z" name: isvc-raw-sklearn-batcher-2c740.18be79f14684c287 namespace: kserve-ci-e2e-test resourceVersion: "14683" uid: 7025847c-0324-4f55-9e0a-7caa60088baa reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-2c740 namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: 14ac2015-7d99-4b89-8baf-2bdec5f85aa2 kind: Event lastTimestamp: "2026-07-02T12:51:19Z" message: InferenceService [isvc-raw-sklearn-batcher-2c740] is Ready metadata: creationTimestamp: "2026-07-02T12:51:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:51:19Z" name: isvc-raw-sklearn-batcher-2c740.18be7a089ff50273 namespace: kserve-ci-e2e-test resourceVersion: "15453" uid: a755b5f9-e79f-47ff-ae9a-f82d8050348f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:01:36.516557Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-c9e60f-predictor-d9496c8c6-fhbcq namespace: kserve-ci-e2e-test resourceVersion: "20751" uid: c698ea02-a654-4343-86af-86adba47fbd8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-c9e60f-predictor-d9496c8c6-fhbcq to ip-10-0-143-88.ec2.internal metadata: creationTimestamp: "2026-07-02T13:01:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:01:36Z" name: isvc-secondary-c9e60f-predictor-d9496c8c6-fhbcq.18be7a982e5a1997 namespace: kserve-ci-e2e-test resourceVersion: "20755" uid: e2c84ea6-404c-4631-955f-dcc39b170508 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-fcf8ff-ccrjf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:36Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-c9e60f-predictor-d9496c8c6-fhbcq namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: c698ea02-a654-4343-86af-86adba47fbd8 kind: Event lastTimestamp: "2026-07-02T13:01:36Z" message: Add eth0 [10.132.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:01:36Z" name: isvc-secondary-c9e60f-predictor-d9496c8c6-fhbcq.18be7a98480cec9f namespace: kserve-ci-e2e-test resourceVersion: "20770" uid: 10d1ac30-1723-40eb-ae5d-513e27b07da0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:01:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-c9e60f-predictor-d9496c8c6-fhbcq namespace: kserve-ci-e2e-test resourceVersion: "20753" uid: c698ea02-a654-4343-86af-86adba47fbd8 kind: Event lastTimestamp: "2026-07-02T13:01:43Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ad817336055c6da6bf66cacbd06b857caf8adb146e60422365f2b38a3444569b" already present on machine metadata: creationTimestamp: "2026-07-02T13:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:43Z" name: isvc-secondary-c9e60f-predictor-d9496c8c6-fhbcq.18be7a98498a7a6b namespace: kserve-ci-e2e-test resourceVersion: "20826" uid: 6322e976-a007-4733-838a-69ea370d226b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:01:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-c9e60f-predictor-d9496c8c6-fhbcq namespace: kserve-ci-e2e-test resourceVersion: "20753" uid: c698ea02-a654-4343-86af-86adba47fbd8 kind: Event lastTimestamp: "2026-07-02T13:01:43Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T13:01:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:43Z" name: isvc-secondary-c9e60f-predictor-d9496c8c6-fhbcq.18be7a984d045048 namespace: kserve-ci-e2e-test resourceVersion: "20829" uid: d7475623-b245-4610-bde0-ad5956a4029c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:01:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-c9e60f-predictor-d9496c8c6-fhbcq namespace: kserve-ci-e2e-test resourceVersion: "20753" uid: c698ea02-a654-4343-86af-86adba47fbd8 kind: Event lastTimestamp: "2026-07-02T13:01:43Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T13:01:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:43Z" name: isvc-secondary-c9e60f-predictor-d9496c8c6-fhbcq.18be7a984d7d0367 namespace: kserve-ci-e2e-test resourceVersion: "20830" uid: fe103c8a-0361-4253-9337-3abd40bf5144 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-c9e60f-predictor-d9496c8c6-fhbcq namespace: kserve-ci-e2e-test resourceVersion: "20753" uid: c698ea02-a654-4343-86af-86adba47fbd8 kind: Event lastTimestamp: "2026-07-02T13:01:48Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-c9e60f-predictor-d9496c8c6-fhbcq_kserve-ci-e2e-test(c698ea02-a654-4343-86af-86adba47fbd8) metadata: creationTimestamp: "2026-07-02T13:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:48Z" name: isvc-secondary-c9e60f-predictor-d9496c8c6-fhbcq.18be7a9b024cdfab namespace: kserve-ci-e2e-test resourceVersion: "20867" uid: 2f72f533-1f3c-4267-a855-88a606ca1a91 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-c9e60f-predictor-d9496c8c6 namespace: kserve-ci-e2e-test resourceVersion: "20739" uid: 42dff183-6556-4398-85c9-ef5d061da963 kind: Event lastTimestamp: "2026-07-02T13:01:36Z" message: 'Created pod: isvc-secondary-c9e60f-predictor-d9496c8c6-fhbcq' metadata: creationTimestamp: "2026-07-02T13:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:01:36Z" name: isvc-secondary-c9e60f-predictor-d9496c8c6.18be7a982de5293a namespace: kserve-ci-e2e-test resourceVersion: "20752" uid: d4abb7ed-acb1-4da9-9632-60fa4b105fd6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-c9e60f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20737" uid: ae3a9746-6f2b-4d49-a189-52d1da1ddfdc kind: Event lastTimestamp: "2026-07-02T13:01:36Z" message: Scaled up replica set isvc-secondary-c9e60f-predictor-d9496c8c6 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:01:36Z" name: isvc-secondary-c9e60f-predictor.18be7a9829062019 namespace: kserve-ci-e2e-test resourceVersion: "20740" uid: 54244b9d-1047-4d27-b9c2-9a2d287e991f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-c9e60f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20738" uid: 04100b79-9d84-452b-85ec-06d2071ce6b7 kind: Event lastTimestamp: "2026-07-02T13:01:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:01:51Z" name: isvc-secondary-c9e60f-predictor.18be7a9ba7c541b4 namespace: kserve-ci-e2e-test resourceVersion: "20893" uid: 59a8ba60-5c17-4952-9c53-e8ff5e0edcd8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-c9e60f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20738" uid: 04100b79-9d84-452b-85ec-06d2071ce6b7 kind: Event lastTimestamp: "2026-07-02T13:01:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:01:51Z" name: isvc-secondary-c9e60f-predictor.18be7a9ba84a6a21 namespace: kserve-ci-e2e-test resourceVersion: "20895" uid: 7ab9f582-2046-4d3c-8564-4785840e4bb5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T13:01:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-c9e60f namespace: kserve-ci-e2e-test resourceVersion: "20730" uid: 209a9029-8e08-4bbe-9d5a-5e5e220e1ff5 kind: Event lastTimestamp: "2026-07-02T13:01:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:01:48Z" name: isvc-secondary-c9e60f.18be7a9829190f43 namespace: kserve-ci-e2e-test resourceVersion: "20873" uid: 3f06b640-3d6c-48cf-8cda-bd2a61cfbb95 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:01:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-c9e60f namespace: kserve-ci-e2e-test resourceVersion: "20730" uid: 209a9029-8e08-4bbe-9d5a-5e5e220e1ff5 kind: Event lastTimestamp: "2026-07-02T13:01:36Z" message: 'Failed to update status for InferenceService "isvc-secondary-c9e60f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-c9e60f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:01:36Z" name: isvc-secondary-c9e60f.18be7a98298a9627 namespace: kserve-ci-e2e-test resourceVersion: "20763" uid: d893a776-1e08-4ff0-a659-b7b1b63b5c4d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:01:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-c9e60f namespace: kserve-ci-e2e-test resourceVersion: "20730" uid: 209a9029-8e08-4bbe-9d5a-5e5e220e1ff5 kind: Event lastTimestamp: "2026-07-02T13:01:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-c9e60f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:01:36Z" name: isvc-secondary-c9e60f.18be7a98298ada83 namespace: kserve-ci-e2e-test resourceVersion: "20764" uid: cedd757e-4d70-420f-9719-5ed12e91a21f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T12:51:34.828601Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns namespace: kserve-ci-e2e-test resourceVersion: "15595" uid: 44bac380-51a6-475a-a4ce-a339dc65a434 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns to ip-10-0-143-88.ec2.internal metadata: creationTimestamp: "2026-07-02T12:51:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T12:51:34Z" name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns.18be7a0c16f46b86 namespace: kserve-ci-e2e-test resourceVersion: "15610" uid: 9a3851ad-9773-47f4-98b8-44e75f5cdb39 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-fcf8ff-ccrjf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:35Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns namespace: kserve-ci-e2e-test resourceVersion: "15615" uid: 44bac380-51a6-475a-a4ce-a339dc65a434 kind: Event lastTimestamp: "2026-07-02T12:51:35Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T12:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T12:51:35Z" name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns.18be7a0c31410985 namespace: kserve-ci-e2e-test resourceVersion: "15652" uid: c4730df0-7217-4e13-828e-53a3b50b47e8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 44bac380-51a6-475a-a4ce-a339dc65a434 kind: Event lastTimestamp: "2026-07-02T12:51:35Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ad817336055c6da6bf66cacbd06b857caf8adb146e60422365f2b38a3444569b" already present on machine metadata: creationTimestamp: "2026-07-02T12:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:35Z" name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns.18be7a0c32d943cd namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: d1bbc50a-d928-4340-8124-31d0050ce38a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 44bac380-51a6-475a-a4ce-a339dc65a434 kind: Event lastTimestamp: "2026-07-02T12:51:35Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T12:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:35Z" name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns.18be7a0c35af780d namespace: kserve-ci-e2e-test resourceVersion: "15655" uid: 03daafaa-5d80-47df-8f7e-5ec751f29960 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 44bac380-51a6-475a-a4ce-a339dc65a434 kind: Event lastTimestamp: "2026-07-02T12:51:35Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T12:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:35Z" name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns.18be7a0c363b8177 namespace: kserve-ci-e2e-test resourceVersion: "15657" uid: 8272e623-ed08-4eba-b943-2dc346047b4b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 44bac380-51a6-475a-a4ce-a339dc65a434 kind: Event lastTimestamp: "2026-07-02T12:51:39Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1694" already present on machine metadata: creationTimestamp: "2026-07-02T12:51:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:39Z" name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns.18be7a0d3408700b namespace: kserve-ci-e2e-test resourceVersion: "15707" uid: d1474f5e-2732-4110-b6d5-550554eb5ccd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 44bac380-51a6-475a-a4ce-a339dc65a434 kind: Event lastTimestamp: "2026-07-02T12:51:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T12:51:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:39Z" name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns.18be7a0d3816d673 namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: e9e0a460-e16f-4c7f-8b74-5be85f44a0b4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 44bac380-51a6-475a-a4ce-a339dc65a434 kind: Event lastTimestamp: "2026-07-02T12:51:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T12:51:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:39Z" name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns.18be7a0d388c9aad namespace: kserve-ci-e2e-test resourceVersion: "15713" uid: 139350b6-33cd-4e97-a9bc-dcda102e0cad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 44bac380-51a6-475a-a4ce-a339dc65a434 kind: Event lastTimestamp: "2026-07-02T12:51:39Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T12:51:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:39Z" name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns.18be7a0d389708ce namespace: kserve-ci-e2e-test resourceVersion: "15714" uid: d51b1ba5-15f1-4daf-9cd1-3661ba78cdac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 44bac380-51a6-475a-a4ce-a339dc65a434 kind: Event lastTimestamp: "2026-07-02T12:51:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T12:51:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:39Z" name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns.18be7a0d3be32b9a namespace: kserve-ci-e2e-test resourceVersion: "15717" uid: f39cc70b-3f3a-434a-9575-a681c7efde56 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 44bac380-51a6-475a-a4ce-a339dc65a434 kind: Event lastTimestamp: "2026-07-02T12:51:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:51:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:39Z" name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns.18be7a0d3c581d84 namespace: kserve-ci-e2e-test resourceVersion: "15718" uid: 052ce971-3096-4314-83e3-cb1858836b1c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T12:51:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 44bac380-51a6-475a-a4ce-a339dc65a434 kind: Event lastTimestamp: "2026-07-02T12:53:37Z" message: 'Readiness probe failed: dial tcp 10.132.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T12:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:37Z" name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns.18be7a0dac5e7f52 namespace: kserve-ci-e2e-test resourceVersion: "16794" uid: a7c22058-4174-4671-b9f2-817938372bbd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 44bac380-51a6-475a-a4ce-a339dc65a434 kind: Event lastTimestamp: "2026-07-02T12:53:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T12:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:35Z" name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns.18be7a28212bf293 namespace: kserve-ci-e2e-test resourceVersion: "16709" uid: b8d780a5-76a4-4264-b234-ff2e5a3ceb48 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 44bac380-51a6-475a-a4ce-a339dc65a434 kind: Event lastTimestamp: "2026-07-02T12:53:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:35Z" name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns.18be7a28212ed6d4 namespace: kserve-ci-e2e-test resourceVersion: "16714" uid: 9fe1766d-9a38-4dd0-9066-7df91b624883 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 44bac380-51a6-475a-a4ce-a339dc65a434 kind: Event lastTimestamp: "2026-07-02T12:53:37Z" message: 'Readiness probe failed: Get "https://10.132.0.38:8643/healthz": dial tcp 10.132.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T12:53:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:37Z" name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns.18be7a28aebb384d namespace: kserve-ci-e2e-test resourceVersion: "16793" uid: 5d831c7f-7938-4305-a042-919fc3a6c3dd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4 namespace: kserve-ci-e2e-test resourceVersion: "15569" uid: f4a95905-016f-4912-a4bd-7432a8fcb544 kind: Event lastTimestamp: "2026-07-02T12:51:34Z" message: 'Created pod: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4-lqjns' metadata: creationTimestamp: "2026-07-02T12:51:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:51:34Z" name: isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4.18be7a0c1689c1ce namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: a4a478a0-0e79-4ba7-b941-ecaa4afdb00e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-e1206-predictor namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: e5db12c4-0110-42dc-aea2-87954e038bda kind: Event lastTimestamp: "2026-07-02T12:51:34Z" message: Scaled up replica set isvc-sklearn-graph-raw-e1206-predictor-666bd7dbf4 from 0 to 1 metadata: creationTimestamp: "2026-07-02T12:51:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:51:34Z" name: isvc-sklearn-graph-raw-e1206-predictor.18be7a0c11f3ea70 namespace: kserve-ci-e2e-test resourceVersion: "15571" uid: 94271ad5-ffeb-40e1-9ba0-5344965ae140 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T12:51:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-e1206-predictor namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: b9b219c8-4198-4b80-b119-8002469c78cc kind: Event lastTimestamp: "2026-07-02T12:52:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:51:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:04Z" name: isvc-sklearn-graph-raw-e1206-predictor.18be7a0f9088162f namespace: kserve-ci-e2e-test resourceVersion: "15914" uid: 77be1f6f-5a0f-4030-b9d7-866140832217 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T12:51:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-e1206-predictor namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: b9b219c8-4198-4b80-b119-8002469c78cc kind: Event lastTimestamp: "2026-07-02T12:52:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:51:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:04Z" name: isvc-sklearn-graph-raw-e1206-predictor.18be7a0f90f099f8 namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: 1f3d1b02-a9e3-449c-ba8f-91d0f4927d10 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T12:52:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-e1206-predictor namespace: kserve-ci-e2e-test resourceVersion: "15797" uid: b9b219c8-4198-4b80-b119-8002469c78cc kind: Event lastTimestamp: "2026-07-02T12:53:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:52:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:53:34Z" name: isvc-sklearn-graph-raw-e1206-predictor.18be7a168edbb1a0 namespace: kserve-ci-e2e-test resourceVersion: "16635" uid: dc26ae03-2c47-488e-b4f4-1f3421f80bb9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T12:52:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-e1206-predictor namespace: kserve-ci-e2e-test resourceVersion: "15797" uid: b9b219c8-4198-4b80-b119-8002469c78cc kind: Event lastTimestamp: "2026-07-02T12:53:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:52:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:53:34Z" name: isvc-sklearn-graph-raw-e1206-predictor.18be7a168f3b4a0b namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: 533f3e72-7ec3-40b9-933f-aed5c86f2192 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T12:51:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-e1206 namespace: kserve-ci-e2e-test resourceVersion: "15558" uid: e6328a1c-4af9-462c-a450-98689677f4e0 kind: Event lastTimestamp: "2026-07-02T12:52:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T12:51:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:52:57Z" name: isvc-sklearn-graph-raw-e1206.18be7a0c11f7fca0 namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: a3731813-197e-46ba-956f-06810f9533fd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-e1206 namespace: kserve-ci-e2e-test resourceVersion: "15558" uid: e6328a1c-4af9-462c-a450-98689677f4e0 kind: Event lastTimestamp: "2026-07-02T12:51:34Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-e1206": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-e1206": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T12:51:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:51:34Z" name: isvc-sklearn-graph-raw-e1206.18be7a0c126a3f2b namespace: kserve-ci-e2e-test resourceVersion: "15573" uid: e18b1044-1df1-401a-84eb-30b80a393021 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-e1206 namespace: kserve-ci-e2e-test resourceVersion: "16318" uid: e6328a1c-4af9-462c-a450-98689677f4e0 kind: Event lastTimestamp: "2026-07-02T12:52:57Z" message: InferenceService [isvc-sklearn-graph-raw-e1206] is Ready metadata: creationTimestamp: "2026-07-02T12:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:52:57Z" name: isvc-sklearn-graph-raw-e1206.18be7a1f663e95d3 namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: f79d2f7a-90a7-4fb0-8166-70caf596f777 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T12:53:35.244352Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx namespace: kserve-ci-e2e-test resourceVersion: "16688" uid: 7866608b-328f-49f4-b8c0-1cfb595e820d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx to ip-10-0-143-88.ec2.internal metadata: creationTimestamp: "2026-07-02T12:53:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T12:53:35Z" name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx.18be7a28204af752 namespace: kserve-ci-e2e-test resourceVersion: "16700" uid: 3f7c4147-7901-4fed-b1c4-e8838dd8f22e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-fcf8ff-ccrjf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:35Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx namespace: kserve-ci-e2e-test resourceVersion: "16712" uid: 7866608b-328f-49f4-b8c0-1cfb595e820d kind: Event lastTimestamp: "2026-07-02T12:53:35Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T12:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T12:53:35Z" name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx.18be7a283a7a6033 namespace: kserve-ci-e2e-test resourceVersion: "16768" uid: cf970e52-0b44-434a-a6f8-e8d657228c01 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx namespace: kserve-ci-e2e-test resourceVersion: "16695" uid: 7866608b-328f-49f4-b8c0-1cfb595e820d kind: Event lastTimestamp: "2026-07-02T12:53:35Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ad817336055c6da6bf66cacbd06b857caf8adb146e60422365f2b38a3444569b" already present on machine metadata: creationTimestamp: "2026-07-02T12:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:35Z" name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx.18be7a283c0ee74f namespace: kserve-ci-e2e-test resourceVersion: "16770" uid: 49ed3d68-718c-4bd6-9337-a7e3ad0de072 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx namespace: kserve-ci-e2e-test resourceVersion: "16695" uid: 7866608b-328f-49f4-b8c0-1cfb595e820d kind: Event lastTimestamp: "2026-07-02T12:53:35Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T12:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:35Z" name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx.18be7a283ee47251 namespace: kserve-ci-e2e-test resourceVersion: "16771" uid: a4612e4f-fa7a-4b13-b2cf-2003e314c9a4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx namespace: kserve-ci-e2e-test resourceVersion: "16695" uid: 7866608b-328f-49f4-b8c0-1cfb595e820d kind: Event lastTimestamp: "2026-07-02T12:53:35Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T12:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:35Z" name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx.18be7a283f5279af namespace: kserve-ci-e2e-test resourceVersion: "16772" uid: 6205cd05-4b32-46b2-b048-6ffb8ddb2ef4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx namespace: kserve-ci-e2e-test resourceVersion: "16695" uid: 7866608b-328f-49f4-b8c0-1cfb595e820d kind: Event lastTimestamp: "2026-07-02T12:53:40Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1694" already present on machine metadata: creationTimestamp: "2026-07-02T12:53:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:40Z" name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx.18be7a2949f1321e namespace: kserve-ci-e2e-test resourceVersion: "16827" uid: 5ce3d85a-f447-4b3b-8177-90350bfd8b46 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx namespace: kserve-ci-e2e-test resourceVersion: "16695" uid: 7866608b-328f-49f4-b8c0-1cfb595e820d kind: Event lastTimestamp: "2026-07-02T12:53:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T12:53:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:40Z" name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx.18be7a294e30c852 namespace: kserve-ci-e2e-test resourceVersion: "16831" uid: 4933d2d9-2003-4834-8ce3-1040f44c2e08 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx namespace: kserve-ci-e2e-test resourceVersion: "16695" uid: 7866608b-328f-49f4-b8c0-1cfb595e820d kind: Event lastTimestamp: "2026-07-02T12:53:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T12:53:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:40Z" name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx.18be7a294e9f405b namespace: kserve-ci-e2e-test resourceVersion: "16835" uid: e02c10bb-ae75-4197-8192-19d64b998220 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx namespace: kserve-ci-e2e-test resourceVersion: "16695" uid: 7866608b-328f-49f4-b8c0-1cfb595e820d kind: Event lastTimestamp: "2026-07-02T12:53:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T12:53:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:40Z" name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx.18be7a294ea8e53d namespace: kserve-ci-e2e-test resourceVersion: "16836" uid: 92848085-d660-4a36-b8cf-ad871af8146c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx namespace: kserve-ci-e2e-test resourceVersion: "16695" uid: 7866608b-328f-49f4-b8c0-1cfb595e820d kind: Event lastTimestamp: "2026-07-02T12:53:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T12:53:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:40Z" name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx.18be7a2951b4e14b namespace: kserve-ci-e2e-test resourceVersion: "16838" uid: 48b5d9d2-4f54-4715-8705-2079328efece reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx namespace: kserve-ci-e2e-test resourceVersion: "16695" uid: 7866608b-328f-49f4-b8c0-1cfb595e820d kind: Event lastTimestamp: "2026-07-02T12:53:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:53:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:40Z" name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx.18be7a295224c56a namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 10282c15-c053-4e09-a644-b973fd685cb7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T12:53:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx namespace: kserve-ci-e2e-test resourceVersion: "16695" uid: 7866608b-328f-49f4-b8c0-1cfb595e820d kind: Event lastTimestamp: "2026-07-02T12:55:27Z" message: 'Readiness probe failed: dial tcp 10.132.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T12:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:27Z" name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx.18be7a29865f9a62 namespace: kserve-ci-e2e-test resourceVersion: "17794" uid: 7fc5eddd-ec69-4ba9-a601-995f7d96ae38 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx namespace: kserve-ci-e2e-test resourceVersion: "16695" uid: 7866608b-328f-49f4-b8c0-1cfb595e820d kind: Event lastTimestamp: "2026-07-02T12:55:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T12:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:25Z" name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx.18be7a41d894189b namespace: kserve-ci-e2e-test resourceVersion: "17748" uid: 286d946b-39dc-469a-9e4f-969d2dd58010 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx namespace: kserve-ci-e2e-test resourceVersion: "16695" uid: 7866608b-328f-49f4-b8c0-1cfb595e820d kind: Event lastTimestamp: "2026-07-02T12:55:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:25Z" name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx.18be7a41d89537d0 namespace: kserve-ci-e2e-test resourceVersion: "17751" uid: 7e7ae197-d19d-4042-84c9-52da8a58d2cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx namespace: kserve-ci-e2e-test resourceVersion: "16695" uid: 7866608b-328f-49f4-b8c0-1cfb595e820d kind: Event lastTimestamp: "2026-07-02T12:55:27Z" message: 'Readiness probe failed: Get "https://10.132.0.40:8643/healthz": dial tcp 10.132.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T12:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:27Z" name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx.18be7a4234c56bbf namespace: kserve-ci-e2e-test resourceVersion: "17793" uid: df8a63ca-b0ea-435c-a9f1-b4a6e1728d2a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746 namespace: kserve-ci-e2e-test resourceVersion: "16669" uid: 4dc9dc3b-4016-4aab-b196-d57e7c4e1813 kind: Event lastTimestamp: "2026-07-02T12:53:35Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746-wx5lx' metadata: creationTimestamp: "2026-07-02T12:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:53:35Z" name: isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746.18be7a281f86cff0 namespace: kserve-ci-e2e-test resourceVersion: "16693" uid: f0666241-e0d6-4979-a192-33bdff027eb5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-9587b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16668" uid: 77974977-a67d-4ff0-9c1a-fa7290a3d1cf kind: Event lastTimestamp: "2026-07-02T12:53:35Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-9587b-predictor-54f88c8746 from 0 to 1 metadata: creationTimestamp: "2026-07-02T12:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:53:35Z" name: isvc-sklearn-graph-raw-hpa-9587b-predictor.18be7a2819644e69 namespace: kserve-ci-e2e-test resourceVersion: "16671" uid: 5727f740-ed84-400a-b06a-589ce2e97a8c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T12:53:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-9587b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16670" uid: 964cb052-d149-4c82-8bed-f610c167109a kind: Event lastTimestamp: "2026-07-02T12:54:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:53:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:54:05Z" name: isvc-sklearn-graph-raw-hpa-9587b-predictor.18be7a2b999af49f namespace: kserve-ci-e2e-test resourceVersion: "17035" uid: 4e5ce399-dc23-489a-af50-b033ba6c431b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T12:53:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-9587b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16670" uid: 964cb052-d149-4c82-8bed-f610c167109a kind: Event lastTimestamp: "2026-07-02T12:54:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:53:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:54:05Z" name: isvc-sklearn-graph-raw-hpa-9587b-predictor.18be7a2b9a41e242 namespace: kserve-ci-e2e-test resourceVersion: "17036" uid: fb92447d-6cb3-41f9-8c4d-9d74d608c867 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T12:54:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-9587b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16926" uid: 964cb052-d149-4c82-8bed-f610c167109a kind: Event lastTimestamp: "2026-07-02T12:55:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:54:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:55:20Z" name: isvc-sklearn-graph-raw-hpa-9587b-predictor.18be7a3298965d85 namespace: kserve-ci-e2e-test resourceVersion: "17646" uid: 27e20650-d184-497c-bd01-7335d266954e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T12:54:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-9587b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16926" uid: 964cb052-d149-4c82-8bed-f610c167109a kind: Event lastTimestamp: "2026-07-02T12:55:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:54:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:55:20Z" name: isvc-sklearn-graph-raw-hpa-9587b-predictor.18be7a329940f6ef namespace: kserve-ci-e2e-test resourceVersion: "17647" uid: 871fb1a4-8f89-463f-9c48-17086e9f9962 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T12:53:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-9587b namespace: kserve-ci-e2e-test resourceVersion: "16658" uid: 664ffef0-432d-484a-a3f6-b135e6090f0a kind: Event lastTimestamp: "2026-07-02T12:54:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T12:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:54:57Z" name: isvc-sklearn-graph-raw-hpa-9587b.18be7a2819f54b66 namespace: kserve-ci-e2e-test resourceVersion: "17436" uid: 56c81af9-ef2e-4d20-8469-c46f4ceff6bb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-9587b namespace: kserve-ci-e2e-test resourceVersion: "16658" uid: 664ffef0-432d-484a-a3f6-b135e6090f0a kind: Event lastTimestamp: "2026-07-02T12:53:35Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-9587b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-9587b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T12:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:53:35Z" name: isvc-sklearn-graph-raw-hpa-9587b.18be7a281ab43b5d namespace: kserve-ci-e2e-test resourceVersion: "16674" uid: d8dcbb31-2d44-445d-b87d-633e4f0111d4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:54:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-9587b namespace: kserve-ci-e2e-test resourceVersion: "17434" uid: 664ffef0-432d-484a-a3f6-b135e6090f0a kind: Event lastTimestamp: "2026-07-02T12:54:57Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-9587b] is Ready metadata: creationTimestamp: "2026-07-02T12:54:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:54:57Z" name: isvc-sklearn-graph-raw-hpa-9587b.18be7a3b4072c151 namespace: kserve-ci-e2e-test resourceVersion: "17435" uid: 28abb8dd-0fad-46e6-8d1d-7dcfcd91da70 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T12:57:20.965373Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc namespace: kserve-ci-e2e-test resourceVersion: "18759" uid: 4774483e-4db5-4cd4-ac2c-cd33a7c33331 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc to ip-10-0-143-88.ec2.internal metadata: creationTimestamp: "2026-07-02T12:57:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T12:57:20Z" name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc.18be7a5cae5070b4 namespace: kserve-ci-e2e-test resourceVersion: "18771" uid: f22fc6f7-35d4-4e22-bcf9-51657627b4fa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-fcf8ff-ccrjf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:57:21Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc namespace: kserve-ci-e2e-test resourceVersion: "18763" uid: 4774483e-4db5-4cd4-ac2c-cd33a7c33331 kind: Event lastTimestamp: "2026-07-02T12:57:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-scale-raw-56395-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T12:57:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:57:21Z" name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc.18be7a5cb594703d namespace: kserve-ci-e2e-test resourceVersion: "18790" uid: 1d773afb-5076-4623-80c6-450562a1920d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:57:21Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc namespace: kserve-ci-e2e-test resourceVersion: "18783" uid: 4774483e-4db5-4cd4-ac2c-cd33a7c33331 kind: Event lastTimestamp: "2026-07-02T12:57:21Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T12:57:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T12:57:21Z" name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc.18be7a5cebdb1ffa namespace: kserve-ci-e2e-test resourceVersion: "18801" uid: 65e48094-b217-497b-8f98-1c8c0db5ba0f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:57:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc namespace: kserve-ci-e2e-test resourceVersion: "18763" uid: 4774483e-4db5-4cd4-ac2c-cd33a7c33331 kind: Event lastTimestamp: "2026-07-02T12:57:22Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ad817336055c6da6bf66cacbd06b857caf8adb146e60422365f2b38a3444569b" already present on machine metadata: creationTimestamp: "2026-07-02T12:57:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:57:22Z" name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc.18be7a5cedfa5be4 namespace: kserve-ci-e2e-test resourceVersion: "18803" uid: 5023212f-6172-4b41-874e-0207bd2eafac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:57:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc namespace: kserve-ci-e2e-test resourceVersion: "18763" uid: 4774483e-4db5-4cd4-ac2c-cd33a7c33331 kind: Event lastTimestamp: "2026-07-02T12:57:22Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T12:57:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:57:22Z" name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc.18be7a5cf11412ed namespace: kserve-ci-e2e-test resourceVersion: "18805" uid: 8d991e2f-5363-48ce-87cd-3f4dcade6bd7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:57:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc namespace: kserve-ci-e2e-test resourceVersion: "18763" uid: 4774483e-4db5-4cd4-ac2c-cd33a7c33331 kind: Event lastTimestamp: "2026-07-02T12:57:22Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T12:57:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:57:22Z" name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc.18be7a5cf1893b52 namespace: kserve-ci-e2e-test resourceVersion: "18806" uid: 694317e3-2444-46e1-835c-ea4180537542 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:57:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc namespace: kserve-ci-e2e-test resourceVersion: "18763" uid: 4774483e-4db5-4cd4-ac2c-cd33a7c33331 kind: Event lastTimestamp: "2026-07-02T12:57:26Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1694" already present on machine metadata: creationTimestamp: "2026-07-02T12:57:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:57:26Z" name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc.18be7a5df0c80ae0 namespace: kserve-ci-e2e-test resourceVersion: "18841" uid: 7c964026-6350-43aa-82c6-577f23af033e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:57:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc namespace: kserve-ci-e2e-test resourceVersion: "18763" uid: 4774483e-4db5-4cd4-ac2c-cd33a7c33331 kind: Event lastTimestamp: "2026-07-02T12:57:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T12:57:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:57:26Z" name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc.18be7a5df410e54e namespace: kserve-ci-e2e-test resourceVersion: "18843" uid: 01f2b22f-e411-4d50-b015-fb96a918615f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:57:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc namespace: kserve-ci-e2e-test resourceVersion: "18763" uid: 4774483e-4db5-4cd4-ac2c-cd33a7c33331 kind: Event lastTimestamp: "2026-07-02T12:57:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T12:57:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:57:26Z" name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc.18be7a5df4858546 namespace: kserve-ci-e2e-test resourceVersion: "18846" uid: f50e81f4-7ea7-483a-a0e3-db9a4c3ac3d1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:57:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc namespace: kserve-ci-e2e-test resourceVersion: "18763" uid: 4774483e-4db5-4cd4-ac2c-cd33a7c33331 kind: Event lastTimestamp: "2026-07-02T12:57:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T12:57:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:57:26Z" name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc.18be7a5df48ee784 namespace: kserve-ci-e2e-test resourceVersion: "18847" uid: 576449f2-2ca2-4063-9106-129fac2aeea9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:57:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc namespace: kserve-ci-e2e-test resourceVersion: "18763" uid: 4774483e-4db5-4cd4-ac2c-cd33a7c33331 kind: Event lastTimestamp: "2026-07-02T12:57:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T12:57:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:57:26Z" name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc.18be7a5df8140d71 namespace: kserve-ci-e2e-test resourceVersion: "18850" uid: 1475d17d-0393-4741-ad99-473ad92991d2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:57:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc namespace: kserve-ci-e2e-test resourceVersion: "18763" uid: 4774483e-4db5-4cd4-ac2c-cd33a7c33331 kind: Event lastTimestamp: "2026-07-02T12:57:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:57:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:57:26Z" name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc.18be7a5df882dab3 namespace: kserve-ci-e2e-test resourceVersion: "18851" uid: e063980f-f910-4ae4-8593-d57bcf27e77a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 19 eventTime: null firstTimestamp: "2026-07-02T12:57:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc namespace: kserve-ci-e2e-test resourceVersion: "18763" uid: 4774483e-4db5-4cd4-ac2c-cd33a7c33331 kind: Event lastTimestamp: "2026-07-02T13:00:09Z" message: 'Readiness probe failed: dial tcp 10.132.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T12:57:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:09Z" name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc.18be7a5e2ce009cc namespace: kserve-ci-e2e-test resourceVersion: "20085" uid: f2f625ec-b1ff-4665-9178-9b14c9aae3dd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc namespace: kserve-ci-e2e-test resourceVersion: "18763" uid: 4774483e-4db5-4cd4-ac2c-cd33a7c33331 kind: Event lastTimestamp: "2026-07-02T13:00:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:00:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:01Z" name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc.18be7a81f93e6f72 namespace: kserve-ci-e2e-test resourceVersion: "19971" uid: 7364b264-bee3-404c-aac7-1fb7a903c686 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc namespace: kserve-ci-e2e-test resourceVersion: "18763" uid: 4774483e-4db5-4cd4-ac2c-cd33a7c33331 kind: Event lastTimestamp: "2026-07-02T13:00:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:00:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:01Z" name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc.18be7a81f93ef245 namespace: kserve-ci-e2e-test resourceVersion: "19973" uid: 51527fa7-e804-4bf3-8d74-411a94f50fb6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:00:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc namespace: kserve-ci-e2e-test resourceVersion: "18763" uid: 4774483e-4db5-4cd4-ac2c-cd33a7c33331 kind: Event lastTimestamp: "2026-07-02T13:00:08Z" message: 'Readiness probe failed: Get "https://10.132.0.43:8643/healthz": dial tcp 10.132.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:00:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:08Z" name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc.18be7a827f6e8452 namespace: kserve-ci-e2e-test resourceVersion: "20072" uid: 734c7e95-08b6-409c-a0a6-77b3675cac76 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:57:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4 namespace: kserve-ci-e2e-test resourceVersion: "18734" uid: d52662d3-018c-40c4-9328-f8a7570473eb kind: Event lastTimestamp: "2026-07-02T12:57:20Z" message: 'Created pod: isvc-sklearn-scale-raw-56395-predictor-5845d646b4-nxtmc' metadata: creationTimestamp: "2026-07-02T12:57:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:57:20Z" name: isvc-sklearn-scale-raw-56395-predictor-5845d646b4.18be7a5cadedd55f namespace: kserve-ci-e2e-test resourceVersion: "18764" uid: 0163a9dd-4255-49cb-a6f2-7baeab776d28 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:57:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-56395-predictor namespace: kserve-ci-e2e-test resourceVersion: "18732" uid: f0dc46ea-4861-4b11-b18f-eb24475d3774 kind: Event lastTimestamp: "2026-07-02T12:57:20Z" message: Scaled up replica set isvc-sklearn-scale-raw-56395-predictor-5845d646b4 from 0 to 1 metadata: creationTimestamp: "2026-07-02T12:57:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:57:20Z" name: isvc-sklearn-scale-raw-56395-predictor.18be7a5ca8b1ce66 namespace: kserve-ci-e2e-test resourceVersion: "18735" uid: 0d8d1263-ff68-4369-b53d-cc8aac6b386f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T12:57:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-56395-predictor namespace: kserve-ci-e2e-test resourceVersion: "18733" uid: 6ce0f0fe-5be8-4729-9800-a8194282bba5 kind: Event lastTimestamp: "2026-07-02T12:57:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:57:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:57:50Z" name: isvc-sklearn-scale-raw-56395-predictor.18be7a6027efb9b5 namespace: kserve-ci-e2e-test resourceVersion: "19038" uid: 79372daa-fc2a-4423-8bf5-324d1242f608 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T12:57:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-56395-predictor namespace: kserve-ci-e2e-test resourceVersion: "18733" uid: 6ce0f0fe-5be8-4729-9800-a8194282bba5 kind: Event lastTimestamp: "2026-07-02T12:57:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:57:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:57:50Z" name: isvc-sklearn-scale-raw-56395-predictor.18be7a60285c8af5 namespace: kserve-ci-e2e-test resourceVersion: "19039" uid: 47acca1c-ba20-4b8c-bf4e-4adc576ac12c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-02T12:58:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-56395-predictor namespace: kserve-ci-e2e-test resourceVersion: "18921" uid: 6ce0f0fe-5be8-4729-9800-a8194282bba5 kind: Event lastTimestamp: "2026-07-02T12:59:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:58:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:59:51Z" name: isvc-sklearn-scale-raw-56395-predictor.18be7a6726fd7c8b namespace: kserve-ci-e2e-test resourceVersion: "19872" uid: 219d7d11-6dc5-4df9-8c02-443a4112b1f9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-02T12:58:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-56395-predictor namespace: kserve-ci-e2e-test resourceVersion: "18921" uid: 6ce0f0fe-5be8-4729-9800-a8194282bba5 kind: Event lastTimestamp: "2026-07-02T12:59:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:58:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:59:51Z" name: isvc-sklearn-scale-raw-56395-predictor.18be7a6727669dd7 namespace: kserve-ci-e2e-test resourceVersion: "19873" uid: 5b738bd6-878d-4127-b74f-1b094a54c290 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T12:57:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-56395 namespace: kserve-ci-e2e-test resourceVersion: "18724" uid: a1d67133-0a0a-4be9-bbaf-ea3976183da7 kind: Event lastTimestamp: "2026-07-02T13:00:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T12:57:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:00:00Z" name: isvc-sklearn-scale-raw-56395.18be7a5ca8b6e6cb namespace: kserve-ci-e2e-test resourceVersion: "19948" uid: 0f61d63e-261f-4217-be34-aaddade730a2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:57:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-56395 namespace: kserve-ci-e2e-test resourceVersion: "18724" uid: a1d67133-0a0a-4be9-bbaf-ea3976183da7 kind: Event lastTimestamp: "2026-07-02T12:57:20Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-56395": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-56395": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T12:57:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:57:20Z" name: isvc-sklearn-scale-raw-56395.18be7a5ca9369439 namespace: kserve-ci-e2e-test resourceVersion: "18738" uid: df9b13de-8359-464b-8e34-43c9276970b9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:59:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-56395 namespace: kserve-ci-e2e-test resourceVersion: "19944" uid: a1d67133-0a0a-4be9-bbaf-ea3976183da7 kind: Event lastTimestamp: "2026-07-02T12:59:59Z" message: InferenceService [isvc-sklearn-scale-raw-56395] is Ready metadata: creationTimestamp: "2026-07-02T12:59:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:59:59Z" name: isvc-sklearn-scale-raw-56395.18be7a81b5341080 namespace: kserve-ci-e2e-test resourceVersion: "19946" uid: 616ee20b-6269-4db4-9b45-d65859dcfcd5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T12:51:34.907274Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz namespace: kserve-ci-e2e-test resourceVersion: "15628" uid: c3450b92-228e-4217-8a8c-0f25d12e1938 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz to ip-10-0-143-88.ec2.internal metadata: creationTimestamp: "2026-07-02T12:51:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T12:51:34Z" name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz.18be7a0c1ba4efe3 namespace: kserve-ci-e2e-test resourceVersion: "15633" uid: 722e7e69-22f3-4f46-bff0-210e3ba1683f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-fcf8ff-ccrjf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:35Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz namespace: kserve-ci-e2e-test resourceVersion: "15638" uid: c3450b92-228e-4217-8a8c-0f25d12e1938 kind: Event lastTimestamp: "2026-07-02T12:51:35Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T12:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T12:51:35Z" name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz.18be7a0c36345f2d namespace: kserve-ci-e2e-test resourceVersion: "15656" uid: 3e1330ca-bc68-4589-a8a2-660e9381ef75 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz namespace: kserve-ci-e2e-test resourceVersion: "15630" uid: c3450b92-228e-4217-8a8c-0f25d12e1938 kind: Event lastTimestamp: "2026-07-02T12:51:35Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ad817336055c6da6bf66cacbd06b857caf8adb146e60422365f2b38a3444569b" already present on machine metadata: creationTimestamp: "2026-07-02T12:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:35Z" name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz.18be7a0c395341ae namespace: kserve-ci-e2e-test resourceVersion: "15659" uid: 86b30f12-793c-4d38-9f30-af74818b154a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz namespace: kserve-ci-e2e-test resourceVersion: "15630" uid: c3450b92-228e-4217-8a8c-0f25d12e1938 kind: Event lastTimestamp: "2026-07-02T12:51:35Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T12:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:35Z" name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz.18be7a0c3c348f38 namespace: kserve-ci-e2e-test resourceVersion: "15660" uid: 71dae0bd-6532-41cd-945a-416177ea95c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz namespace: kserve-ci-e2e-test resourceVersion: "15630" uid: c3450b92-228e-4217-8a8c-0f25d12e1938 kind: Event lastTimestamp: "2026-07-02T12:51:35Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T12:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:35Z" name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz.18be7a0c3ca5b0ed namespace: kserve-ci-e2e-test resourceVersion: "15661" uid: 1a003412-18a0-45f0-8fed-96279bbb5815 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz namespace: kserve-ci-e2e-test resourceVersion: "15630" uid: c3450b92-228e-4217-8a8c-0f25d12e1938 kind: Event lastTimestamp: "2026-07-02T12:51:39Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-02T12:51:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:39Z" name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz.18be7a0d342107aa namespace: kserve-ci-e2e-test resourceVersion: "15708" uid: e45d6d78-e02e-418f-8d2a-1a62200564c3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz namespace: kserve-ci-e2e-test resourceVersion: "15630" uid: c3450b92-228e-4217-8a8c-0f25d12e1938 kind: Event lastTimestamp: "2026-07-02T12:52:00Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 20.697s (20.697s including waiting). Image size: 1305293048 bytes.' metadata: creationTimestamp: "2026-07-02T12:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:00Z" name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz.18be7a1205c7db50 namespace: kserve-ci-e2e-test resourceVersion: "15875" uid: 5e95661d-1b77-4a09-8cf5-861ee2236f50 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz namespace: kserve-ci-e2e-test resourceVersion: "15630" uid: c3450b92-228e-4217-8a8c-0f25d12e1938 kind: Event lastTimestamp: "2026-07-02T12:52:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T12:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:00Z" name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz.18be7a12093dd845 namespace: kserve-ci-e2e-test resourceVersion: "15876" uid: be85b8db-8127-48eb-92cf-459dd9149087 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz namespace: kserve-ci-e2e-test resourceVersion: "15630" uid: c3450b92-228e-4217-8a8c-0f25d12e1938 kind: Event lastTimestamp: "2026-07-02T12:52:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T12:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:00Z" name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz.18be7a1209b207c1 namespace: kserve-ci-e2e-test resourceVersion: "15877" uid: 4745b6db-e1a4-40cb-8cc0-f527edc487b7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz namespace: kserve-ci-e2e-test resourceVersion: "15630" uid: c3450b92-228e-4217-8a8c-0f25d12e1938 kind: Event lastTimestamp: "2026-07-02T12:52:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T12:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:00Z" name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz.18be7a1209bb6734 namespace: kserve-ci-e2e-test resourceVersion: "15878" uid: be726629-958b-474e-a52b-aaf219d9a340 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz namespace: kserve-ci-e2e-test resourceVersion: "15630" uid: c3450b92-228e-4217-8a8c-0f25d12e1938 kind: Event lastTimestamp: "2026-07-02T12:52:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T12:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:00Z" name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz.18be7a120cbc961c namespace: kserve-ci-e2e-test resourceVersion: "15879" uid: 1642def4-27c1-42e7-be14-3081e99df719 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz namespace: kserve-ci-e2e-test resourceVersion: "15630" uid: c3450b92-228e-4217-8a8c-0f25d12e1938 kind: Event lastTimestamp: "2026-07-02T12:52:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:00Z" name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz.18be7a120d49e43d namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: 421651c4-2805-450b-bd0b-688a99739992 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T12:52:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz namespace: kserve-ci-e2e-test resourceVersion: "15630" uid: c3450b92-228e-4217-8a8c-0f25d12e1938 kind: Event lastTimestamp: "2026-07-02T12:53:36Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T12:52:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:36Z" name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz.18be7a121f7d21c7 namespace: kserve-ci-e2e-test resourceVersion: "16787" uid: 87b00e7c-47ae-46bd-bceb-b1aff492cde0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz namespace: kserve-ci-e2e-test resourceVersion: "15630" uid: c3450b92-228e-4217-8a8c-0f25d12e1938 kind: Event lastTimestamp: "2026-07-02T12:53:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T12:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:35Z" name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz.18be7a282ada0621 namespace: kserve-ci-e2e-test resourceVersion: "16754" uid: 052d2473-32bc-42be-b9ec-5bd6f7092e51 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz namespace: kserve-ci-e2e-test resourceVersion: "15630" uid: c3450b92-228e-4217-8a8c-0f25d12e1938 kind: Event lastTimestamp: "2026-07-02T12:53:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:35Z" name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz.18be7a282ada7191 namespace: kserve-ci-e2e-test resourceVersion: "16756" uid: c472e1ec-2c5a-448e-a208-b6ca185e09c4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz namespace: kserve-ci-e2e-test resourceVersion: "15630" uid: c3450b92-228e-4217-8a8c-0f25d12e1938 kind: Event lastTimestamp: "2026-07-02T12:53:36Z" message: 'Readiness probe failed: Get "https://10.132.0.39:8643/healthz": dial tcp 10.132.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T12:53:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:36Z" name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz.18be7a2879c74f4f namespace: kserve-ci-e2e-test resourceVersion: "16786" uid: 0eaea07f-90a5-4187-82be-4b0dca72b8f2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc namespace: kserve-ci-e2e-test resourceVersion: "15613" uid: 3df4ec85-c2c2-4652-9f11-6ba812632bb8 kind: Event lastTimestamp: "2026-07-02T12:51:34Z" message: 'Created pod: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc-x6dvz' metadata: creationTimestamp: "2026-07-02T12:51:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:51:34Z" name: isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc.18be7a0c1b3b48d9 namespace: kserve-ci-e2e-test resourceVersion: "15629" uid: 7016f1a7-c182-412a-ad97-253e04794b99 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-e1206-predictor namespace: kserve-ci-e2e-test resourceVersion: "15607" uid: e8546b10-4a98-4426-9abf-3a45614ddf15 kind: Event lastTimestamp: "2026-07-02T12:51:34Z" message: Scaled up replica set isvc-xgboost-graph-raw-e1206-predictor-66d5466ccc from 0 to 1 metadata: creationTimestamp: "2026-07-02T12:51:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:51:34Z" name: isvc-xgboost-graph-raw-e1206-predictor.18be7a0c1859dac2 namespace: kserve-ci-e2e-test resourceVersion: "15616" uid: 00548c11-e1c7-461c-ac51-41c1d717d962 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:51:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-e1206-predictor namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: a4f0285e-416b-4647-848a-e274df9e24a2 kind: Event lastTimestamp: "2026-07-02T12:52:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:51:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:19Z" name: isvc-xgboost-graph-raw-e1206-predictor.18be7a0f96d5cc91 namespace: kserve-ci-e2e-test resourceVersion: "16036" uid: 1dea7e5a-1fe5-4abf-ae8c-b0513850363c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:51:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-e1206-predictor namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: a4f0285e-416b-4647-848a-e274df9e24a2 kind: Event lastTimestamp: "2026-07-02T12:52:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:51:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:19Z" name: isvc-xgboost-graph-raw-e1206-predictor.18be7a0f9738218c namespace: kserve-ci-e2e-test resourceVersion: "16037" uid: 0fb83551-5300-4274-896a-f5e65d9eb501 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T12:52:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-e1206-predictor namespace: kserve-ci-e2e-test resourceVersion: "15801" uid: a4f0285e-416b-4647-848a-e274df9e24a2 kind: Event lastTimestamp: "2026-07-02T12:53:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:52:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:53:34Z" name: isvc-xgboost-graph-raw-e1206-predictor.18be7a1a136e1831 namespace: kserve-ci-e2e-test resourceVersion: "16652" uid: 5e9fc1df-19ac-4f31-a8e2-de17635300d6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T12:52:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-e1206-predictor namespace: kserve-ci-e2e-test resourceVersion: "15801" uid: a4f0285e-416b-4647-848a-e274df9e24a2 kind: Event lastTimestamp: "2026-07-02T12:53:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:52:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:53:34Z" name: isvc-xgboost-graph-raw-e1206-predictor.18be7a1a13d74c1c namespace: kserve-ci-e2e-test resourceVersion: "16653" uid: 9d0ebf0e-e9d5-4abf-a81d-3d96124d470a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T12:51:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-e1206 namespace: kserve-ci-e2e-test resourceVersion: "15575" uid: 22801e85-33fb-4b27-8004-e1b5a9f53b43 kind: Event lastTimestamp: "2026-07-02T12:53:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T12:51:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:53:06Z" name: isvc-xgboost-graph-raw-e1206.18be7a0c18b2fc14 namespace: kserve-ci-e2e-test resourceVersion: "16392" uid: a2a6058f-b630-49f4-aba9-29e13053f287 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-e1206 namespace: kserve-ci-e2e-test resourceVersion: "15575" uid: 22801e85-33fb-4b27-8004-e1b5a9f53b43 kind: Event lastTimestamp: "2026-07-02T12:51:34Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-e1206": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-e1206": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T12:51:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:51:34Z" name: isvc-xgboost-graph-raw-e1206.18be7a0c19073b12 namespace: kserve-ci-e2e-test resourceVersion: "15619" uid: dfec625c-e134-4f40-bb08-de4e95f4f375 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-e1206 namespace: kserve-ci-e2e-test resourceVersion: "16390" uid: 22801e85-33fb-4b27-8004-e1b5a9f53b43 kind: Event lastTimestamp: "2026-07-02T12:53:06Z" message: InferenceService [isvc-xgboost-graph-raw-e1206] is Ready metadata: creationTimestamp: "2026-07-02T12:53:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:53:06Z" name: isvc-xgboost-graph-raw-e1206.18be7a21867172dd namespace: kserve-ci-e2e-test resourceVersion: "16391" uid: 2aba460d-7fc2-43ec-abc4-f5e2dbc61cc0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T12:53:35.358972Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv namespace: kserve-ci-e2e-test resourceVersion: "16723" uid: 710d975e-9b70-4ec3-a60d-f80a2782a582 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv to ip-10-0-142-167.ec2.internal metadata: creationTimestamp: "2026-07-02T12:53:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T12:53:35Z" name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv.18be7a28271ff8ec namespace: kserve-ci-e2e-test resourceVersion: "16732" uid: 1b3065ec-c52e-4348-b556-5bb75cdfa047 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-fcf8ff-ccrjf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:35Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv namespace: kserve-ci-e2e-test resourceVersion: "16747" uid: 710d975e-9b70-4ec3-a60d-f80a2782a582 kind: Event lastTimestamp: "2026-07-02T12:53:35Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T12:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T12:53:35Z" name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv.18be7a2841c3bce9 namespace: kserve-ci-e2e-test resourceVersion: "16773" uid: 5fcc6170-e6c4-4f2a-80f5-f9ac832ce632 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv namespace: kserve-ci-e2e-test resourceVersion: "16727" uid: 710d975e-9b70-4ec3-a60d-f80a2782a582 kind: Event lastTimestamp: "2026-07-02T12:53:35Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:ad817336055c6da6bf66cacbd06b857caf8adb146e60422365f2b38a3444569b" metadata: creationTimestamp: "2026-07-02T12:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:35Z" name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv.18be7a284339353a namespace: kserve-ci-e2e-test resourceVersion: "16775" uid: 8e0041d2-b12e-4bab-a6b4-b8cfdae654c7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv namespace: kserve-ci-e2e-test resourceVersion: "16727" uid: 710d975e-9b70-4ec3-a60d-f80a2782a582 kind: Event lastTimestamp: "2026-07-02T12:53:39Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:ad817336055c6da6bf66cacbd06b857caf8adb146e60422365f2b38a3444569b" in 3.261s (3.261s including waiting). Image size: 301878719 bytes.' metadata: creationTimestamp: "2026-07-02T12:53:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:39Z" name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv.18be7a29059c2fab namespace: kserve-ci-e2e-test resourceVersion: "16806" uid: b7550350-dcdd-4e9c-b86d-120537b623fa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv namespace: kserve-ci-e2e-test resourceVersion: "16727" uid: 710d975e-9b70-4ec3-a60d-f80a2782a582 kind: Event lastTimestamp: "2026-07-02T12:53:39Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T12:53:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:39Z" name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv.18be7a290880fbc4 namespace: kserve-ci-e2e-test resourceVersion: "16807" uid: fd8601d1-4e87-4aaa-bab2-bff47f8cd158 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv namespace: kserve-ci-e2e-test resourceVersion: "16727" uid: 710d975e-9b70-4ec3-a60d-f80a2782a582 kind: Event lastTimestamp: "2026-07-02T12:53:39Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T12:53:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:39Z" name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv.18be7a2908eefe4a namespace: kserve-ci-e2e-test resourceVersion: "16808" uid: 7ce8940d-462d-46fe-a99d-9d2c8351fd50 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv namespace: kserve-ci-e2e-test resourceVersion: "16727" uid: 710d975e-9b70-4ec3-a60d-f80a2782a582 kind: Event lastTimestamp: "2026-07-02T12:53:43Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-02T12:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:43Z" name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv.18be7a2a20624559 namespace: kserve-ci-e2e-test resourceVersion: "16868" uid: 4471d2e0-05dc-4d25-868f-8d56f1260722 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:54:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv namespace: kserve-ci-e2e-test resourceVersion: "16727" uid: 710d975e-9b70-4ec3-a60d-f80a2782a582 kind: Event lastTimestamp: "2026-07-02T12:54:03Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 19.758s (19.758s including waiting). Image size: 1305293048 bytes.' metadata: creationTimestamp: "2026-07-02T12:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:54:03Z" name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv.18be7a2eba100fb2 namespace: kserve-ci-e2e-test resourceVersion: "17020" uid: 3ea895df-a3e8-49bd-80b4-4e41b1482b9e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:54:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv namespace: kserve-ci-e2e-test resourceVersion: "16727" uid: 710d975e-9b70-4ec3-a60d-f80a2782a582 kind: Event lastTimestamp: "2026-07-02T12:54:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T12:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:54:03Z" name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv.18be7a2ebcf459a9 namespace: kserve-ci-e2e-test resourceVersion: "17021" uid: 06609708-9b71-4ecb-b41e-29dc41a60076 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:54:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv namespace: kserve-ci-e2e-test resourceVersion: "16727" uid: 710d975e-9b70-4ec3-a60d-f80a2782a582 kind: Event lastTimestamp: "2026-07-02T12:54:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T12:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:54:03Z" name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv.18be7a2ebd67cc39 namespace: kserve-ci-e2e-test resourceVersion: "17023" uid: ce8696cd-647e-41b4-a5d2-78e0bc9a7f1e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:54:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv namespace: kserve-ci-e2e-test resourceVersion: "16727" uid: 710d975e-9b70-4ec3-a60d-f80a2782a582 kind: Event lastTimestamp: "2026-07-02T12:54:03Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T12:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:54:03Z" name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv.18be7a2ebd75bed2 namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: e886c73a-3d61-4d60-8222-75930a1919ea reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:54:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv namespace: kserve-ci-e2e-test resourceVersion: "16727" uid: 710d975e-9b70-4ec3-a60d-f80a2782a582 kind: Event lastTimestamp: "2026-07-02T12:54:06Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.461s (2.461s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T12:54:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:54:06Z" name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv.18be7a2f50282f03 namespace: kserve-ci-e2e-test resourceVersion: "17048" uid: 3173ad5c-6176-421c-b7c5-d020226e62bd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:54:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv namespace: kserve-ci-e2e-test resourceVersion: "16727" uid: 710d975e-9b70-4ec3-a60d-f80a2782a582 kind: Event lastTimestamp: "2026-07-02T12:54:06Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T12:54:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:54:06Z" name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv.18be7a2f5389ac57 namespace: kserve-ci-e2e-test resourceVersion: "17050" uid: d41adcdc-2828-449a-a11b-cf74d21acd41 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:54:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv namespace: kserve-ci-e2e-test resourceVersion: "16727" uid: 710d975e-9b70-4ec3-a60d-f80a2782a582 kind: Event lastTimestamp: "2026-07-02T12:54:06Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:54:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:54:06Z" name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv.18be7a2f53f9dc3d namespace: kserve-ci-e2e-test resourceVersion: "17051" uid: 45b1a14e-071a-4e9b-8cc1-55f2eb0aab26 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T12:54:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv namespace: kserve-ci-e2e-test resourceVersion: "16727" uid: 710d975e-9b70-4ec3-a60d-f80a2782a582 kind: Event lastTimestamp: "2026-07-02T12:54:52Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T12:54:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:54:52Z" name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv.18be7a2f815de98c namespace: kserve-ci-e2e-test resourceVersion: "17398" uid: 0bce6376-fc1c-4b2b-940a-1332fabcc07b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv namespace: kserve-ci-e2e-test resourceVersion: "16727" uid: 710d975e-9b70-4ec3-a60d-f80a2782a582 kind: Event lastTimestamp: "2026-07-02T12:55:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T12:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:25Z" name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv.18be7a41dcbedfb7 namespace: kserve-ci-e2e-test resourceVersion: "17771" uid: 1ebfa4fe-34f4-40f0-ab77-b6998af91abd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv namespace: kserve-ci-e2e-test resourceVersion: "16727" uid: 710d975e-9b70-4ec3-a60d-f80a2782a582 kind: Event lastTimestamp: "2026-07-02T12:55:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:25Z" name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv.18be7a41dcbf7744 namespace: kserve-ci-e2e-test resourceVersion: "17772" uid: d9b5e0b8-e0d5-4205-84fa-1bc5d0252813 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv namespace: kserve-ci-e2e-test resourceVersion: "16727" uid: 710d975e-9b70-4ec3-a60d-f80a2782a582 kind: Event lastTimestamp: "2026-07-02T12:55:27Z" message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": dial tcp 10.134.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T12:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:27Z" name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv.18be7a425d9b2813 namespace: kserve-ci-e2e-test resourceVersion: "17800" uid: d28f89d9-8562-4016-b181-2c420fc9bb68 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5 namespace: kserve-ci-e2e-test resourceVersion: "16711" uid: dcf97208-3bd7-4c8c-b857-4d6a7e629249 kind: Event lastTimestamp: "2026-07-02T12:53:35Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5-jlfhv' metadata: creationTimestamp: "2026-07-02T12:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:53:35Z" name: isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5.18be7a282664105c namespace: kserve-ci-e2e-test resourceVersion: "16726" uid: 213e7abf-be1e-4608-951f-ba2ebca512de reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-9587b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16706" uid: feabf705-2ba4-4620-bcaa-554efabf733c kind: Event lastTimestamp: "2026-07-02T12:53:35Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-9587b-predictor-fb57bb5b5 from 0 to 1 metadata: creationTimestamp: "2026-07-02T12:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:53:35Z" name: isvc-xgboost-graph-raw-hpa-9587b-predictor.18be7a282227a69f namespace: kserve-ci-e2e-test resourceVersion: "16713" uid: 36ba5906-3a45-4013-9573-f8261c21efa6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T12:53:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-9587b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16710" uid: 176e7a2e-09a6-48a8-be8c-2f6895552d78 kind: Event lastTimestamp: "2026-07-02T12:54:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:53:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:54:35Z" name: isvc-xgboost-graph-raw-hpa-9587b-predictor.18be7a2ba166fe29 namespace: kserve-ci-e2e-test resourceVersion: "17264" uid: 58f40e18-0c6f-4b10-ab0e-1e84eaa70597 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T12:53:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-9587b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16710" uid: 176e7a2e-09a6-48a8-be8c-2f6895552d78 kind: Event lastTimestamp: "2026-07-02T12:54:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:53:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:54:35Z" name: isvc-xgboost-graph-raw-hpa-9587b-predictor.18be7a2ba237c4ef namespace: kserve-ci-e2e-test resourceVersion: "17265" uid: 6a508d91-db06-46c8-9b6d-5b13baa56b93 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:54:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-9587b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16928" uid: 176e7a2e-09a6-48a8-be8c-2f6895552d78 kind: Event lastTimestamp: "2026-07-02T12:55:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:54:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:55:20Z" name: isvc-xgboost-graph-raw-hpa-9587b-predictor.18be7a399d755354 namespace: kserve-ci-e2e-test resourceVersion: "17649" uid: 47b37e06-3025-4eae-a216-b7ae56b3ba19 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:54:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-9587b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16928" uid: 176e7a2e-09a6-48a8-be8c-2f6895552d78 kind: Event lastTimestamp: "2026-07-02T12:55:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:54:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:55:20Z" name: isvc-xgboost-graph-raw-hpa-9587b-predictor.18be7a399df731b4 namespace: kserve-ci-e2e-test resourceVersion: "17651" uid: b09685a2-5536-456b-b7bf-81b7da12923f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T12:53:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-9587b namespace: kserve-ci-e2e-test resourceVersion: "16676" uid: 6e0246a1-d7e9-4ed1-b972-cc9e5ebad3ea kind: Event lastTimestamp: "2026-07-02T12:55:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T12:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:55:03Z" name: isvc-xgboost-graph-raw-hpa-9587b.18be7a282289e9b4 namespace: kserve-ci-e2e-test resourceVersion: "17487" uid: 2c5b826b-556f-48a7-b416-8c5a95f94c72 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-9587b namespace: kserve-ci-e2e-test resourceVersion: "16676" uid: 6e0246a1-d7e9-4ed1-b972-cc9e5ebad3ea kind: Event lastTimestamp: "2026-07-02T12:53:35Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-hpa-9587b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-hpa-9587b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T12:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:53:35Z" name: isvc-xgboost-graph-raw-hpa-9587b.18be7a28238a4089 namespace: kserve-ci-e2e-test resourceVersion: "16717" uid: f6437ff0-411d-4796-b577-2c0ee6d92df1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-9587b namespace: kserve-ci-e2e-test resourceVersion: "17485" uid: 6e0246a1-d7e9-4ed1-b972-cc9e5ebad3ea kind: Event lastTimestamp: "2026-07-02T12:55:03Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-9587b] is Ready metadata: creationTimestamp: "2026-07-02T12:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:55:03Z" name: isvc-xgboost-graph-raw-hpa-9587b.18be7a3c9265cef8 namespace: kserve-ci-e2e-test resourceVersion: "17486" uid: f71a3503-7ad4-41bc-9427-69a71713d5c4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T12:55:25.673050Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-c99c3-predictor-7f55f4f59c-vm9qv namespace: kserve-ci-e2e-test resourceVersion: "17731" uid: 15064fb3-c271-4243-9c62-6bb1be858fc2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-c99c3-predictor-7f55f4f59c-vm9qv to ip-10-0-143-88.ec2.internal metadata: creationTimestamp: "2026-07-02T12:55:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T12:55:25Z" name: message-dumper-raw-c99c3-predictor-7f55f4f59c-vm9qv.18be7a41d65b3746 namespace: kserve-ci-e2e-test resourceVersion: "17740" uid: 78aa8ed9-7e61-43e0-8c4d-6def52f7a6cd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-fcf8ff-ccrjf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:25Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-c99c3-predictor-7f55f4f59c-vm9qv namespace: kserve-ci-e2e-test resourceVersion: "17735" uid: 15064fb3-c271-4243-9c62-6bb1be858fc2 kind: Event lastTimestamp: "2026-07-02T12:55:25Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "message-dumper-raw-c99c3-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T12:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:25Z" name: message-dumper-raw-c99c3-predictor-7f55f4f59c-vm9qv.18be7a41e2c73c7f namespace: kserve-ci-e2e-test resourceVersion: "17775" uid: 8600fe22-d05f-4301-852b-783e1c6bcdbd reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:26Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-c99c3-predictor-7f55f4f59c-vm9qv namespace: kserve-ci-e2e-test resourceVersion: "17750" uid: 15064fb3-c271-4243-9c62-6bb1be858fc2 kind: Event lastTimestamp: "2026-07-02T12:55:26Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T12:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T12:55:26Z" name: message-dumper-raw-c99c3-predictor-7f55f4f59c-vm9qv.18be7a4213eae878 namespace: kserve-ci-e2e-test resourceVersion: "17785" uid: 1b02f6e6-d61f-4ccd-832c-2095b299238a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-c99c3-predictor-7f55f4f59c-vm9qv namespace: kserve-ci-e2e-test resourceVersion: "17735" uid: 15064fb3-c271-4243-9c62-6bb1be858fc2 kind: Event lastTimestamp: "2026-07-02T12:55:26Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-02T12:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:26Z" name: message-dumper-raw-c99c3-predictor-7f55f4f59c-vm9qv.18be7a421558d5ac namespace: kserve-ci-e2e-test resourceVersion: "17787" uid: 1288e343-0241-433a-ac11-6674ea3d4211 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-c99c3-predictor-7f55f4f59c-vm9qv namespace: kserve-ci-e2e-test resourceVersion: "17735" uid: 15064fb3-c271-4243-9c62-6bb1be858fc2 kind: Event lastTimestamp: "2026-07-02T12:55:28Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.285s (1.285s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-02T12:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:28Z" name: message-dumper-raw-c99c3-predictor-7f55f4f59c-vm9qv.18be7a4261fdb5db namespace: kserve-ci-e2e-test resourceVersion: "17802" uid: 675f3bf8-69c5-40ac-889b-dbb742d35e52 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-c99c3-predictor-7f55f4f59c-vm9qv namespace: kserve-ci-e2e-test resourceVersion: "17735" uid: 15064fb3-c271-4243-9c62-6bb1be858fc2 kind: Event lastTimestamp: "2026-07-02T12:55:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T12:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:28Z" name: message-dumper-raw-c99c3-predictor-7f55f4f59c-vm9qv.18be7a4266a9280a namespace: kserve-ci-e2e-test resourceVersion: "17804" uid: 1f9ff229-23c2-4801-a284-af369c1322c1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-c99c3-predictor-7f55f4f59c-vm9qv namespace: kserve-ci-e2e-test resourceVersion: "17735" uid: 15064fb3-c271-4243-9c62-6bb1be858fc2 kind: Event lastTimestamp: "2026-07-02T12:55:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T12:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:28Z" name: message-dumper-raw-c99c3-predictor-7f55f4f59c-vm9qv.18be7a4267471491 namespace: kserve-ci-e2e-test resourceVersion: "17805" uid: 609976c2-3893-4254-a8f2-ccae40fda281 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-c99c3-predictor-7f55f4f59c-vm9qv namespace: kserve-ci-e2e-test resourceVersion: "17735" uid: 15064fb3-c271-4243-9c62-6bb1be858fc2 kind: Event lastTimestamp: "2026-07-02T12:55:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T12:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:28Z" name: message-dumper-raw-c99c3-predictor-7f55f4f59c-vm9qv.18be7a4267534ccd namespace: kserve-ci-e2e-test resourceVersion: "17806" uid: 7336d3be-472a-46c7-97e9-b6a2edbc93dc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-c99c3-predictor-7f55f4f59c-vm9qv namespace: kserve-ci-e2e-test resourceVersion: "17735" uid: 15064fb3-c271-4243-9c62-6bb1be858fc2 kind: Event lastTimestamp: "2026-07-02T12:55:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T12:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:28Z" name: message-dumper-raw-c99c3-predictor-7f55f4f59c-vm9qv.18be7a426a889e9e namespace: kserve-ci-e2e-test resourceVersion: "17807" uid: aaa1c940-c080-4fe9-b0df-8bc4fbe5aeeb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-c99c3-predictor-7f55f4f59c-vm9qv namespace: kserve-ci-e2e-test resourceVersion: "17735" uid: 15064fb3-c271-4243-9c62-6bb1be858fc2 kind: Event lastTimestamp: "2026-07-02T12:55:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:28Z" name: message-dumper-raw-c99c3-predictor-7f55f4f59c-vm9qv.18be7a426af592e8 namespace: kserve-ci-e2e-test resourceVersion: "17808" uid: bfbe5a19-84c1-4ed5-b42b-d99225382091 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:57:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-c99c3-predictor-7f55f4f59c-vm9qv namespace: kserve-ci-e2e-test resourceVersion: "17735" uid: 15064fb3-c271-4243-9c62-6bb1be858fc2 kind: Event lastTimestamp: "2026-07-02T12:57:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T12:57:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:57:21Z" name: message-dumper-raw-c99c3-predictor-7f55f4f59c-vm9qv.18be7a5cb06f8215 namespace: kserve-ci-e2e-test resourceVersion: "18784" uid: 882f6159-13d9-4ad6-a1ff-e98a90842602 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:57:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-c99c3-predictor-7f55f4f59c-vm9qv namespace: kserve-ci-e2e-test resourceVersion: "17735" uid: 15064fb3-c271-4243-9c62-6bb1be858fc2 kind: Event lastTimestamp: "2026-07-02T12:57:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:57:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:57:21Z" name: message-dumper-raw-c99c3-predictor-7f55f4f59c-vm9qv.18be7a5cb070d96c namespace: kserve-ci-e2e-test resourceVersion: "18785" uid: c17783fe-6be7-4494-b13e-a7f611e0a8ed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-c99c3-predictor-7f55f4f59c namespace: kserve-ci-e2e-test resourceVersion: "17716" uid: 7b0c00f6-3ce5-4d82-bea1-50fcd34c308a kind: Event lastTimestamp: "2026-07-02T12:55:25Z" message: 'Created pod: message-dumper-raw-c99c3-predictor-7f55f4f59c-vm9qv' metadata: creationTimestamp: "2026-07-02T12:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:55:25Z" name: message-dumper-raw-c99c3-predictor-7f55f4f59c.18be7a41d5bf71c1 namespace: kserve-ci-e2e-test resourceVersion: "17737" uid: df7764a3-51ef-4aa7-bc39-0553bbe1bc29 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-c99c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "17714" uid: 18c4f468-3435-487c-bc4f-f0ca7c873e51 kind: Event lastTimestamp: "2026-07-02T12:55:25Z" message: Scaled up replica set message-dumper-raw-c99c3-predictor-7f55f4f59c from 0 to 1 metadata: creationTimestamp: "2026-07-02T12:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:55:25Z" name: message-dumper-raw-c99c3-predictor.18be7a41d0861b26 namespace: kserve-ci-e2e-test resourceVersion: "17718" uid: 95b995b6-6f62-426e-9369-444439525c70 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-c99c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "17715" uid: 8c300a60-2ced-499a-9a52-58b2728d78a3 kind: Event lastTimestamp: "2026-07-02T12:55:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:55:40Z" name: message-dumper-raw-c99c3-predictor.18be7a454fb740d2 namespace: kserve-ci-e2e-test resourceVersion: "17917" uid: 03cd1a2d-6668-48d2-b316-74727b4fb241 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-c99c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "17715" uid: 8c300a60-2ced-499a-9a52-58b2728d78a3 kind: Event lastTimestamp: "2026-07-02T12:55:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:55:40Z" name: message-dumper-raw-c99c3-predictor.18be7a4550d87685 namespace: kserve-ci-e2e-test resourceVersion: "17919" uid: fd89171f-a729-4d58-831f-d407517b6c17 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-c99c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "17918" uid: 8c300a60-2ced-499a-9a52-58b2728d78a3 kind: Event lastTimestamp: "2026-07-02T12:55:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:55:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:55:55Z" name: message-dumper-raw-c99c3-predictor.18be7a48d049eab5 namespace: kserve-ci-e2e-test resourceVersion: "18088" uid: a8278afc-1262-425e-b821-e485fcebef62 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-c99c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "17918" uid: 8c300a60-2ced-499a-9a52-58b2728d78a3 kind: Event lastTimestamp: "2026-07-02T12:55:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:55:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:55:55Z" name: message-dumper-raw-c99c3-predictor.18be7a48d0b8e017 namespace: kserve-ci-e2e-test resourceVersion: "18090" uid: 2c189c3d-fafd-4d90-a73b-ab2f127d9f39 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T12:55:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-c99c3 namespace: kserve-ci-e2e-test resourceVersion: "17707" uid: f06e365a-7dc3-4ffc-a322-e4147406c258 kind: Event lastTimestamp: "2026-07-02T12:55:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T12:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:55:36Z" name: message-dumper-raw-c99c3.18be7a41d08951d3 namespace: kserve-ci-e2e-test resourceVersion: "17883" uid: dd837686-a75e-4474-a2c2-9c8914b22135 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-c99c3 namespace: kserve-ci-e2e-test resourceVersion: "17707" uid: f06e365a-7dc3-4ffc-a322-e4147406c258 kind: Event lastTimestamp: "2026-07-02T12:55:25Z" message: 'Failed to update status for InferenceService "message-dumper-raw-c99c3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-c99c3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T12:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:55:25Z" name: message-dumper-raw-c99c3.18be7a41d151284d namespace: kserve-ci-e2e-test resourceVersion: "17721" uid: e8acdcc8-aafc-4373-b061-3187db684643 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-c99c3 namespace: kserve-ci-e2e-test resourceVersion: "17879" uid: f06e365a-7dc3-4ffc-a322-e4147406c258 kind: Event lastTimestamp: "2026-07-02T12:55:35Z" message: InferenceService [message-dumper-raw-c99c3] is Ready metadata: creationTimestamp: "2026-07-02T12:55:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:55:35Z" name: message-dumper-raw-c99c3.18be7a443b0f290b namespace: kserve-ci-e2e-test resourceVersion: "17881" uid: 56b3c61d-3c6f-49c6-bf57-f1c4316e3209 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T12:53:14.858995Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-e1206-678748c8b8-vpv7g namespace: kserve-ci-e2e-test resourceVersion: "16455" uid: ce64604b-2ef0-4f59-907e-66ec836f443f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-e1206-678748c8b8-vpv7g to ip-10-0-142-167.ec2.internal metadata: creationTimestamp: "2026-07-02T12:53:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T12:53:14Z" name: model-chainer-raw-e1206-678748c8b8-vpv7g.18be7a23613b1300 namespace: kserve-ci-e2e-test resourceVersion: "16461" uid: 758379a4-761a-45c1-aa91-ed5aa12c8d6f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-fcf8ff-ccrjf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:15Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-e1206-678748c8b8-vpv7g namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: ce64604b-2ef0-4f59-907e-66ec836f443f kind: Event lastTimestamp: "2026-07-02T12:53:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-e1206-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T12:53:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:15Z" name: model-chainer-raw-e1206-678748c8b8-vpv7g.18be7a236aba63f6 namespace: kserve-ci-e2e-test resourceVersion: "16471" uid: b37d5c60-27e3-4540-929d-e7959e348e2e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:15Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-e1206-678748c8b8-vpv7g namespace: kserve-ci-e2e-test resourceVersion: "16465" uid: ce64604b-2ef0-4f59-907e-66ec836f443f kind: Event lastTimestamp: "2026-07-02T12:53:15Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T12:53:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T12:53:15Z" name: model-chainer-raw-e1206-678748c8b8-vpv7g.18be7a239e322f2b namespace: kserve-ci-e2e-test resourceVersion: "16479" uid: c3b92210-52ec-4faa-b497-f88e56b62987 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-e1206} kind: Pod name: model-chainer-raw-e1206-678748c8b8-vpv7g namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: ce64604b-2ef0-4f59-907e-66ec836f443f kind: Event lastTimestamp: "2026-07-02T12:53:15Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:03660399b01ea350e32a900800c331d43d890f82d1c053ba75b58e21480c1066" metadata: creationTimestamp: "2026-07-02T12:53:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:15Z" name: model-chainer-raw-e1206-678748c8b8-vpv7g.18be7a239f85adc1 namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: 1a2a5826-2cb5-4406-ae76-9e17febd7287 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-e1206} kind: Pod name: model-chainer-raw-e1206-678748c8b8-vpv7g namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: ce64604b-2ef0-4f59-907e-66ec836f443f kind: Event lastTimestamp: "2026-07-02T12:53:18Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:03660399b01ea350e32a900800c331d43d890f82d1c053ba75b58e21480c1066" in 2.222s (2.222s including waiting). Image size: 218070116 bytes.' metadata: creationTimestamp: "2026-07-02T12:53:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:18Z" name: model-chainer-raw-e1206-678748c8b8-vpv7g.18be7a2424047c83 namespace: kserve-ci-e2e-test resourceVersion: "16499" uid: 852dcff8-60a6-449c-99ed-ffe2c6752e56 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-e1206} kind: Pod name: model-chainer-raw-e1206-678748c8b8-vpv7g namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: ce64604b-2ef0-4f59-907e-66ec836f443f kind: Event lastTimestamp: "2026-07-02T12:53:18Z" message: 'Created container: model-chainer-raw-e1206' metadata: creationTimestamp: "2026-07-02T12:53:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:18Z" name: model-chainer-raw-e1206-678748c8b8-vpv7g.18be7a242738dad0 namespace: kserve-ci-e2e-test resourceVersion: "16500" uid: a561c242-5f02-4d1f-aeef-94a5fd91f1d0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-e1206} kind: Pod name: model-chainer-raw-e1206-678748c8b8-vpv7g namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: ce64604b-2ef0-4f59-907e-66ec836f443f kind: Event lastTimestamp: "2026-07-02T12:53:18Z" message: Started container model-chainer-raw-e1206 metadata: creationTimestamp: "2026-07-02T12:53:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:18Z" name: model-chainer-raw-e1206-678748c8b8-vpv7g.18be7a2427ae42d6 namespace: kserve-ci-e2e-test resourceVersion: "16501" uid: b7d17234-0064-4584-aded-a2948b66f8f4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-e1206} kind: Pod name: model-chainer-raw-e1206-678748c8b8-vpv7g namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: ce64604b-2ef0-4f59-907e-66ec836f443f kind: Event lastTimestamp: "2026-07-02T12:53:34Z" message: Stopping container model-chainer-raw-e1206 metadata: creationTimestamp: "2026-07-02T12:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:34Z" name: model-chainer-raw-e1206-678748c8b8-vpv7g.18be7a280e7af44f namespace: kserve-ci-e2e-test resourceVersion: "16650" uid: a3c38827-73b9-4a2e-9d68-fefa5909111a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T12:53:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-e1206} kind: Pod name: model-chainer-raw-e1206-678748c8b8-vpv7g namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: ce64604b-2ef0-4f59-907e-66ec836f443f kind: Event lastTimestamp: "2026-07-02T12:54:04Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T12:53:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:54:04Z" name: model-chainer-raw-e1206-678748c8b8-vpv7g.18be7a292c83541a namespace: kserve-ci-e2e-test resourceVersion: "17032" uid: bd619fb2-8777-4b2c-abd0-bf62d99f8787 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-e1206-678748c8b8 namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: ed678844-0ee7-4864-b0fb-481ca486207e kind: Event lastTimestamp: "2026-07-02T12:53:14Z" message: 'Created pod: model-chainer-raw-e1206-678748c8b8-vpv7g' metadata: creationTimestamp: "2026-07-02T12:53:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:53:14Z" name: model-chainer-raw-e1206-678748c8b8.18be7a2360cfa363 namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: 24a703d4-9767-4054-b62b-65089df11c18 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-e1206 namespace: kserve-ci-e2e-test resourceVersion: "16450" uid: cf95da25-5e84-468c-ae1e-cde3a7657aad kind: Event lastTimestamp: "2026-07-02T12:53:14Z" message: Scaled up replica set model-chainer-raw-e1206-678748c8b8 from 0 to 1 metadata: creationTimestamp: "2026-07-02T12:53:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:53:14Z" name: model-chainer-raw-e1206.18be7a235f96a1c5 namespace: kserve-ci-e2e-test resourceVersion: "16453" uid: 6418f0ea-38b0-4875-a126-edc32b764b89 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:14Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-e1206 namespace: kserve-ci-e2e-test resourceVersion: "16467" uid: 7b85f559-0ea2-4c87-bf4e-4d64dfc889ed kind: Event lastTimestamp: "2026-07-02T12:53:14Z" message: InferenceGraph [model-chainer-raw-e1206] is Ready metadata: creationTimestamp: "2026-07-02T12:53:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:53:14Z" name: model-chainer-raw-e1206.18be7a2364e96919 namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: b2056803-6c35-44ea-9b59-008f12c07659 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: model-chainer-raw-e1206 namespace: kserve-ci-e2e-test resourceVersion: "16451" uid: 205dacce-312e-49e8-a8be-61824c18c8fc kind: Event lastTimestamp: "2026-07-02T12:53:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:53:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:53:29Z" name: model-chainer-raw-e1206.18be7a26de73c339 namespace: kserve-ci-e2e-test resourceVersion: "16596" uid: c5045e70-20af-43da-b8ad-47318b2e3d16 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: model-chainer-raw-e1206 namespace: kserve-ci-e2e-test resourceVersion: "16451" uid: 205dacce-312e-49e8-a8be-61824c18c8fc kind: Event lastTimestamp: "2026-07-02T12:53:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:53:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:53:29Z" name: model-chainer-raw-e1206.18be7a26dedbdb33 namespace: kserve-ci-e2e-test resourceVersion: "16598" uid: e6f72b13-5ab4-43a0-b5a7-1232fc19ce21 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T12:55:15.300558Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-9587b-6fb4ff7f8f-kzn68 namespace: kserve-ci-e2e-test resourceVersion: "17587" uid: a1cdedf6-3d52-4820-8762-7571987f4823 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-9587b-6fb4ff7f8f-kzn68 to ip-10-0-142-167.ec2.internal metadata: creationTimestamp: "2026-07-02T12:55:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T12:55:15Z" name: model-chainer-raw-hpa-9587b-6fb4ff7f8f-kzn68.18be7a3f6c1b8d5d namespace: kserve-ci-e2e-test resourceVersion: "17592" uid: 24478424-afe0-4768-869a-2349365a361a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-fcf8ff-ccrjf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:15Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-9587b-6fb4ff7f8f-kzn68 namespace: kserve-ci-e2e-test resourceVersion: "17591" uid: a1cdedf6-3d52-4820-8762-7571987f4823 kind: Event lastTimestamp: "2026-07-02T12:55:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-hpa-9587b-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T12:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:15Z" name: model-chainer-raw-hpa-9587b-6fb4ff7f8f-kzn68.18be7a3f751a9bd5 namespace: kserve-ci-e2e-test resourceVersion: "17601" uid: d817649e-07f1-475c-ac2f-59923df01217 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:16Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-9587b-6fb4ff7f8f-kzn68 namespace: kserve-ci-e2e-test resourceVersion: "17596" uid: a1cdedf6-3d52-4820-8762-7571987f4823 kind: Event lastTimestamp: "2026-07-02T12:55:16Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T12:55:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T12:55:16Z" name: model-chainer-raw-hpa-9587b-6fb4ff7f8f-kzn68.18be7a3fa916aa7e namespace: kserve-ci-e2e-test resourceVersion: "17611" uid: 7e3c8eed-5c2e-4998-8e6a-5ebfee046730 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-9587b} kind: Pod name: model-chainer-raw-hpa-9587b-6fb4ff7f8f-kzn68 namespace: kserve-ci-e2e-test resourceVersion: "17591" uid: a1cdedf6-3d52-4820-8762-7571987f4823 kind: Event lastTimestamp: "2026-07-02T12:55:16Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:03660399b01ea350e32a900800c331d43d890f82d1c053ba75b58e21480c1066" already present on machine metadata: creationTimestamp: "2026-07-02T12:55:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:16Z" name: model-chainer-raw-hpa-9587b-6fb4ff7f8f-kzn68.18be7a3faa77684d namespace: kserve-ci-e2e-test resourceVersion: "17613" uid: deea5278-e97c-4acc-be40-c4111cd42d64 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-9587b} kind: Pod name: model-chainer-raw-hpa-9587b-6fb4ff7f8f-kzn68 namespace: kserve-ci-e2e-test resourceVersion: "17591" uid: a1cdedf6-3d52-4820-8762-7571987f4823 kind: Event lastTimestamp: "2026-07-02T12:55:16Z" message: 'Created container: model-chainer-raw-hpa-9587b' metadata: creationTimestamp: "2026-07-02T12:55:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:16Z" name: model-chainer-raw-hpa-9587b-6fb4ff7f8f-kzn68.18be7a3fad7a4d43 namespace: kserve-ci-e2e-test resourceVersion: "17615" uid: a4e8a7fa-27ef-4a34-8762-e9da5c35a79e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-9587b} kind: Pod name: model-chainer-raw-hpa-9587b-6fb4ff7f8f-kzn68 namespace: kserve-ci-e2e-test resourceVersion: "17591" uid: a1cdedf6-3d52-4820-8762-7571987f4823 kind: Event lastTimestamp: "2026-07-02T12:55:16Z" message: Started container model-chainer-raw-hpa-9587b metadata: creationTimestamp: "2026-07-02T12:55:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:16Z" name: model-chainer-raw-hpa-9587b-6fb4ff7f8f-kzn68.18be7a3faddebbca namespace: kserve-ci-e2e-test resourceVersion: "17616" uid: e86dde8c-5e8c-4e23-b5ed-079334b55a2f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-9587b} kind: Pod name: model-chainer-raw-hpa-9587b-6fb4ff7f8f-kzn68 namespace: kserve-ci-e2e-test resourceVersion: "17591" uid: a1cdedf6-3d52-4820-8762-7571987f4823 kind: Event lastTimestamp: "2026-07-02T12:55:25Z" message: Stopping container model-chainer-raw-hpa-9587b metadata: creationTimestamp: "2026-07-02T12:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:25Z" name: model-chainer-raw-hpa-9587b-6fb4ff7f8f-kzn68.18be7a41c2b9a036 namespace: kserve-ci-e2e-test resourceVersion: "17700" uid: dbc4f3d0-66a7-436f-a498-c2a47e5dcbf0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T12:55:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-9587b} kind: Pod name: model-chainer-raw-hpa-9587b-6fb4ff7f8f-kzn68 namespace: kserve-ci-e2e-test resourceVersion: "17591" uid: a1cdedf6-3d52-4820-8762-7571987f4823 kind: Event lastTimestamp: "2026-07-02T12:55:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T12:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:55:53Z" name: model-chainer-raw-hpa-9587b-6fb4ff7f8f-kzn68.18be7a426d232a81 namespace: kserve-ci-e2e-test resourceVersion: "18071" uid: 8e13db18-ee91-46b0-89f5-79ec9cf60c5f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-167.ec2.internal source: component: kubelet host: ip-10-0-142-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-9587b-6fb4ff7f8f namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 58cff542-1d52-4699-9198-8ee17b737320 kind: Event lastTimestamp: "2026-07-02T12:55:15Z" message: 'Created pod: model-chainer-raw-hpa-9587b-6fb4ff7f8f-kzn68' metadata: creationTimestamp: "2026-07-02T12:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:55:15Z" name: model-chainer-raw-hpa-9587b-6fb4ff7f8f.18be7a3f6ace10c2 namespace: kserve-ci-e2e-test resourceVersion: "17588" uid: 156c6e57-3a32-4d66-a955-6170f1373603 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-9587b namespace: kserve-ci-e2e-test resourceVersion: "17581" uid: 30710fc9-b944-489f-b352-5e1e8173a15a kind: Event lastTimestamp: "2026-07-02T12:55:15Z" message: Scaled up replica set model-chainer-raw-hpa-9587b-6fb4ff7f8f from 0 to 1 metadata: creationTimestamp: "2026-07-02T12:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:55:15Z" name: model-chainer-raw-hpa-9587b.18be7a3f6993ea85 namespace: kserve-ci-e2e-test resourceVersion: "17585" uid: a7841158-2a9b-4f3d-9317-81f4af52bb2e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:55:15Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-9587b namespace: kserve-ci-e2e-test resourceVersion: "17598" uid: a3af4d3a-b9f8-4964-8b8f-73089e032beb kind: Event lastTimestamp: "2026-07-02T12:55:15Z" message: InferenceGraph [model-chainer-raw-hpa-9587b] is Ready metadata: creationTimestamp: "2026-07-02T12:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:55:15Z" name: model-chainer-raw-hpa-9587b.18be7a3f7273e805 namespace: kserve-ci-e2e-test resourceVersion: "17600" uid: c15b9e02-9236-42e4-bea3-0d27953e0356 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:02:09.985333Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q namespace: kserve-ci-e2e-test resourceVersion: "21174" uid: b240a6aa-163e-41b8-9423-e4e5cd8ed447 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q to ip-10-0-143-88.ec2.internal metadata: creationTimestamp: "2026-07-02T13:02:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:02:09Z" name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q.18be7a9ff93f176c namespace: kserve-ci-e2e-test resourceVersion: "21180" uid: d05050f1-8de4-470c-92b3-8dfbc23c92df reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-fcf8ff-ccrjf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:10Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q namespace: kserve-ci-e2e-test resourceVersion: "21176" uid: b240a6aa-163e-41b8-9423-e4e5cd8ed447 kind: Event lastTimestamp: "2026-07-02T13:02:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-abd6e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:02:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:10Z" name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q.18be7aa006320f9f namespace: kserve-ci-e2e-test resourceVersion: "21189" uid: d45f7b84-0939-4e6b-8871-65b85aa597e4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:11Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q namespace: kserve-ci-e2e-test resourceVersion: "21184" uid: b240a6aa-163e-41b8-9423-e4e5cd8ed447 kind: Event lastTimestamp: "2026-07-02T13:02:11Z" message: Add eth0 [10.132.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:02:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:02:11Z" name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q.18be7aa036a1a57b namespace: kserve-ci-e2e-test resourceVersion: "21197" uid: 8f8672c5-7171-4875-89a9-3380f7a7e300 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q namespace: kserve-ci-e2e-test resourceVersion: "21176" uid: b240a6aa-163e-41b8-9423-e4e5cd8ed447 kind: Event lastTimestamp: "2026-07-02T13:02:11Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ad817336055c6da6bf66cacbd06b857caf8adb146e60422365f2b38a3444569b" already present on machine metadata: creationTimestamp: "2026-07-02T13:02:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:11Z" name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q.18be7aa0383687aa namespace: kserve-ci-e2e-test resourceVersion: "21199" uid: 051a8bf8-1f61-4cfc-8081-a7eef3f40f09 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q namespace: kserve-ci-e2e-test resourceVersion: "21176" uid: b240a6aa-163e-41b8-9423-e4e5cd8ed447 kind: Event lastTimestamp: "2026-07-02T13:02:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T13:02:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:11Z" name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q.18be7aa03b2725c5 namespace: kserve-ci-e2e-test resourceVersion: "21200" uid: 043815a3-1917-429a-8a5a-d01b5057ac5c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q namespace: kserve-ci-e2e-test resourceVersion: "21176" uid: b240a6aa-163e-41b8-9423-e4e5cd8ed447 kind: Event lastTimestamp: "2026-07-02T13:02:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T13:02:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:11Z" name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q.18be7aa03b994f49 namespace: kserve-ci-e2e-test resourceVersion: "21201" uid: 12c825d4-9a5e-4635-aa0d-ff6e416db77c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q namespace: kserve-ci-e2e-test resourceVersion: "21176" uid: b240a6aa-163e-41b8-9423-e4e5cd8ed447 kind: Event lastTimestamp: "2026-07-02T13:02:14Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1694" already present on machine metadata: creationTimestamp: "2026-07-02T13:02:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:14Z" name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q.18be7aa11861e5b2 namespace: kserve-ci-e2e-test resourceVersion: "21232" uid: 143b6d54-15ac-4b00-90e4-9911a34e8824 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q namespace: kserve-ci-e2e-test resourceVersion: "21176" uid: b240a6aa-163e-41b8-9423-e4e5cd8ed447 kind: Event lastTimestamp: "2026-07-02T13:02:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:02:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:14Z" name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q.18be7aa11b627bd9 namespace: kserve-ci-e2e-test resourceVersion: "21235" uid: 016d149e-e094-44b3-8578-9fdaabbc46d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q namespace: kserve-ci-e2e-test resourceVersion: "21176" uid: b240a6aa-163e-41b8-9423-e4e5cd8ed447 kind: Event lastTimestamp: "2026-07-02T13:02:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:02:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:14Z" name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q.18be7aa11bd6350b namespace: kserve-ci-e2e-test resourceVersion: "21236" uid: 9a2b0b46-dd03-469f-bbdc-a5746dc0a48b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q namespace: kserve-ci-e2e-test resourceVersion: "21176" uid: b240a6aa-163e-41b8-9423-e4e5cd8ed447 kind: Event lastTimestamp: "2026-07-02T13:02:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:02:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:14Z" name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q.18be7aa11be064e8 namespace: kserve-ci-e2e-test resourceVersion: "21238" uid: 23279ace-752a-43aa-8e57-42e423ff28d7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q namespace: kserve-ci-e2e-test resourceVersion: "21176" uid: b240a6aa-163e-41b8-9423-e4e5cd8ed447 kind: Event lastTimestamp: "2026-07-02T13:02:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:02:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:14Z" name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q.18be7aa11f2792df namespace: kserve-ci-e2e-test resourceVersion: "21240" uid: 3f965466-82b8-4fae-92ec-8e1dab843a12 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q namespace: kserve-ci-e2e-test resourceVersion: "21176" uid: b240a6aa-163e-41b8-9423-e4e5cd8ed447 kind: Event lastTimestamp: "2026-07-02T13:02:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:02:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:14Z" name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q.18be7aa11f9590c5 namespace: kserve-ci-e2e-test resourceVersion: "21242" uid: 239c77f6-781c-47fe-a702-0e4a762aa081 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T13:02:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q namespace: kserve-ci-e2e-test resourceVersion: "21176" uid: b240a6aa-163e-41b8-9423-e4e5cd8ed447 kind: Event lastTimestamp: "2026-07-02T13:03:41Z" message: 'Readiness probe failed: dial tcp 10.132.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:02:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:41Z" name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q.18be7aa1546c3814 namespace: kserve-ci-e2e-test resourceVersion: "21877" uid: 31a54ed1-3556-46f4-a91d-3a83f5921701 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q namespace: kserve-ci-e2e-test resourceVersion: "21176" uid: b240a6aa-163e-41b8-9423-e4e5cd8ed447 kind: Event lastTimestamp: "2026-07-02T13:03:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:03:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:40Z" name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q.18be7ab4f8e7b95e namespace: kserve-ci-e2e-test resourceVersion: "21861" uid: f4e0d223-59e0-4d5a-8a1d-e7c069f46649 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q namespace: kserve-ci-e2e-test resourceVersion: "21176" uid: b240a6aa-163e-41b8-9423-e4e5cd8ed447 kind: Event lastTimestamp: "2026-07-02T13:03:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:03:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:40Z" name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q.18be7ab4f8e85741 namespace: kserve-ci-e2e-test resourceVersion: "21862" uid: d1af5d02-a7e1-4319-9118-678f880b5d8a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q namespace: kserve-ci-e2e-test resourceVersion: "21176" uid: b240a6aa-163e-41b8-9423-e4e5cd8ed447 kind: Event lastTimestamp: "2026-07-02T13:03:41Z" message: 'Readiness probe failed: Get "https://10.132.0.47:8643/healthz": dial tcp 10.132.0.47:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:03:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:41Z" name: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q.18be7ab55ac03ecc namespace: kserve-ci-e2e-test resourceVersion: "21876" uid: 740dd964-a913-45f9-8599-ce751895d612 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-abd6e-predictor-568bb8fbf8 namespace: kserve-ci-e2e-test resourceVersion: "21165" uid: 1fdc7922-6daa-45de-93f5-f87776fde4ca kind: Event lastTimestamp: "2026-07-02T13:02:09Z" message: 'Created pod: raw-sklearn-abd6e-predictor-568bb8fbf8-d7n2q' metadata: creationTimestamp: "2026-07-02T13:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:02:09Z" name: raw-sklearn-abd6e-predictor-568bb8fbf8.18be7a9ff8c97d38 namespace: kserve-ci-e2e-test resourceVersion: "21178" uid: 4e280226-1f11-4b29-91ca-be3c3ecbb9b5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-abd6e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: 0222115e-f1d9-4e50-babb-a0ef360b6a22 kind: Event lastTimestamp: "2026-07-02T13:02:09Z" message: Scaled up replica set raw-sklearn-abd6e-predictor-568bb8fbf8 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:02:09Z" name: raw-sklearn-abd6e-predictor.18be7a9ff5254828 namespace: kserve-ci-e2e-test resourceVersion: "21166" uid: a620fb32-ea37-4742-8828-552b3e6f4903 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:02:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-abd6e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21164" uid: 6e423bb3-fc49-4f48-81e1-a447eba132a3 kind: Event lastTimestamp: "2026-07-02T13:02:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:02:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:02:39Z" name: raw-sklearn-abd6e-predictor.18be7aa373ee9e0d namespace: kserve-ci-e2e-test resourceVersion: "21421" uid: 3f94a4ef-4bdc-4ced-9359-935bc91ffb0c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:02:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-abd6e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21164" uid: 6e423bb3-fc49-4f48-81e1-a447eba132a3 kind: Event lastTimestamp: "2026-07-02T13:02:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:02:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:02:39Z" name: raw-sklearn-abd6e-predictor.18be7aa37451d1a8 namespace: kserve-ci-e2e-test resourceVersion: "21422" uid: 42cfd0dd-c67f-4cdc-a128-ed0cbe5f3ff4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:02:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-abd6e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21317" uid: 6e423bb3-fc49-4f48-81e1-a447eba132a3 kind: Event lastTimestamp: "2026-07-02T13:03:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:03:40Z" name: raw-sklearn-abd6e-predictor.18be7aaa72531432 namespace: kserve-ci-e2e-test resourceVersion: "21843" uid: 32420dcc-8cae-4337-9446-ee3352c5b032 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:02:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-abd6e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21317" uid: 6e423bb3-fc49-4f48-81e1-a447eba132a3 kind: Event lastTimestamp: "2026-07-02T13:03:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:02:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:03:40Z" name: raw-sklearn-abd6e-predictor.18be7aaa72b88798 namespace: kserve-ci-e2e-test resourceVersion: "21844" uid: 887910fe-b424-403d-a93b-c0673dfcaf1b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T13:02:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-abd6e namespace: kserve-ci-e2e-test resourceVersion: "21156" uid: f817d52b-813e-4438-82f8-2a361ea6c998 kind: Event lastTimestamp: "2026-07-02T13:03:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:03:31Z" name: raw-sklearn-abd6e.18be7a9ff538c4ef namespace: kserve-ci-e2e-test resourceVersion: "21790" uid: c55f7fb3-b9df-426b-8994-2e6d391b1aa0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-abd6e namespace: kserve-ci-e2e-test resourceVersion: "21156" uid: f817d52b-813e-4438-82f8-2a361ea6c998 kind: Event lastTimestamp: "2026-07-02T13:02:09Z" message: 'Failed to update status for InferenceService "raw-sklearn-abd6e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-abd6e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:02:09Z" name: raw-sklearn-abd6e.18be7a9ff5c6416b namespace: kserve-ci-e2e-test resourceVersion: "21170" uid: 2243c8e7-2818-47e0-8524-91d195514328 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-abd6e namespace: kserve-ci-e2e-test resourceVersion: "21787" uid: f817d52b-813e-4438-82f8-2a361ea6c998 kind: Event lastTimestamp: "2026-07-02T13:03:31Z" message: InferenceService [raw-sklearn-abd6e] is Ready metadata: creationTimestamp: "2026-07-02T13:03:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:03:31Z" name: raw-sklearn-abd6e.18be7ab30d8a7513 namespace: kserve-ci-e2e-test resourceVersion: "21788" uid: 9b30db3c-8138-4b58-82fd-4dfe5e26a07c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:03:45.384596Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4 namespace: kserve-ci-e2e-test resourceVersion: "21929" uid: d5b25188-d9d1-4261-b781-d93cbfcb2dc7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4 to ip-10-0-143-88.ec2.internal metadata: creationTimestamp: "2026-07-02T13:03:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:03:45Z" name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4.18be7ab62f7c4879 namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: f2a76ea4-37d3-4fdc-a2e3-170140841c27 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-fcf8ff-ccrjf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:45Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4 namespace: kserve-ci-e2e-test resourceVersion: "21942" uid: d5b25188-d9d1-4261-b781-d93cbfcb2dc7 kind: Event lastTimestamp: "2026-07-02T13:03:45Z" message: Add eth0 [10.132.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:03:45Z" name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4.18be7ab64945dc4d namespace: kserve-ci-e2e-test resourceVersion: "21948" uid: 26ba4a6d-b0ab-4945-a523-f9cb3e40bfb8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4 namespace: kserve-ci-e2e-test resourceVersion: "21932" uid: d5b25188-d9d1-4261-b781-d93cbfcb2dc7 kind: Event lastTimestamp: "2026-07-02T13:03:45Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ad817336055c6da6bf66cacbd06b857caf8adb146e60422365f2b38a3444569b" already present on machine metadata: creationTimestamp: "2026-07-02T13:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:45Z" name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4.18be7ab64adbaac0 namespace: kserve-ci-e2e-test resourceVersion: "21950" uid: 321543df-f572-41ab-ac13-b103f3a403e6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4 namespace: kserve-ci-e2e-test resourceVersion: "21932" uid: d5b25188-d9d1-4261-b781-d93cbfcb2dc7 kind: Event lastTimestamp: "2026-07-02T13:03:45Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T13:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:45Z" name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4.18be7ab64dc07bd5 namespace: kserve-ci-e2e-test resourceVersion: "21951" uid: d83cac98-ca0f-4fe3-b053-40935ca759f0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4 namespace: kserve-ci-e2e-test resourceVersion: "21932" uid: d5b25188-d9d1-4261-b781-d93cbfcb2dc7 kind: Event lastTimestamp: "2026-07-02T13:03:45Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T13:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:45Z" name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4.18be7ab64e892ff2 namespace: kserve-ci-e2e-test resourceVersion: "21952" uid: a078bd0d-c6a8-4762-8719-bf15f94fa11d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4 namespace: kserve-ci-e2e-test resourceVersion: "21932" uid: d5b25188-d9d1-4261-b781-d93cbfcb2dc7 kind: Event lastTimestamp: "2026-07-02T13:03:50Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1694" already present on machine metadata: creationTimestamp: "2026-07-02T13:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:50Z" name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4.18be7ab7531ef100 namespace: kserve-ci-e2e-test resourceVersion: "21992" uid: c5b183d1-608b-48a4-af2f-1610e3b2c39d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4 namespace: kserve-ci-e2e-test resourceVersion: "21932" uid: d5b25188-d9d1-4261-b781-d93cbfcb2dc7 kind: Event lastTimestamp: "2026-07-02T13:03:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:50Z" name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4.18be7ab7566b9f0b namespace: kserve-ci-e2e-test resourceVersion: "21994" uid: 52f4253a-e516-4b6f-aecd-8ec0fdf43a8d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4 namespace: kserve-ci-e2e-test resourceVersion: "21932" uid: d5b25188-d9d1-4261-b781-d93cbfcb2dc7 kind: Event lastTimestamp: "2026-07-02T13:03:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:50Z" name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4.18be7ab756ffdedd namespace: kserve-ci-e2e-test resourceVersion: "21995" uid: 40e2c13e-cb88-4c66-bb92-b04f613be383 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4 namespace: kserve-ci-e2e-test resourceVersion: "21932" uid: d5b25188-d9d1-4261-b781-d93cbfcb2dc7 kind: Event lastTimestamp: "2026-07-02T13:03:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:50Z" name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4.18be7ab7570b6a5e namespace: kserve-ci-e2e-test resourceVersion: "21996" uid: dbde8fbe-65e0-4d38-bad8-171c50b54958 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4 namespace: kserve-ci-e2e-test resourceVersion: "21932" uid: d5b25188-d9d1-4261-b781-d93cbfcb2dc7 kind: Event lastTimestamp: "2026-07-02T13:03:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:50Z" name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4.18be7ab75a81cafc namespace: kserve-ci-e2e-test resourceVersion: "22000" uid: 16aaef81-d932-4463-8f09-e7ac0616142c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4 namespace: kserve-ci-e2e-test resourceVersion: "21932" uid: d5b25188-d9d1-4261-b781-d93cbfcb2dc7 kind: Event lastTimestamp: "2026-07-02T13:03:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:50Z" name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4.18be7ab75af99577 namespace: kserve-ci-e2e-test resourceVersion: "22001" uid: fed5f52d-e916-44e3-afec-8ba1b2d7f758 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T13:03:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4 namespace: kserve-ci-e2e-test resourceVersion: "21932" uid: d5b25188-d9d1-4261-b781-d93cbfcb2dc7 kind: Event lastTimestamp: "2026-07-02T13:05:17Z" message: 'Readiness probe failed: dial tcp 10.132.0.48:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:05:17Z" name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4.18be7ab78f29a50c namespace: kserve-ci-e2e-test resourceVersion: "22650" uid: 41da82d4-08fa-4aa8-900b-1b65d9fca130 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:05:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4 namespace: kserve-ci-e2e-test resourceVersion: "21932" uid: d5b25188-d9d1-4261-b781-d93cbfcb2dc7 kind: Event lastTimestamp: "2026-07-02T13:05:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:05:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:05:15Z" name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4.18be7acb22aa2c2e namespace: kserve-ci-e2e-test resourceVersion: "22632" uid: fc928e07-a376-48fa-bc5c-eab39b908ee3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:05:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4 namespace: kserve-ci-e2e-test resourceVersion: "21932" uid: d5b25188-d9d1-4261-b781-d93cbfcb2dc7 kind: Event lastTimestamp: "2026-07-02T13:05:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:05:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:05:15Z" name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4.18be7acb22aab731 namespace: kserve-ci-e2e-test resourceVersion: "22633" uid: 488c637c-b1a5-4560-80d5-8e98b545dee7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:05:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4 namespace: kserve-ci-e2e-test resourceVersion: "21932" uid: d5b25188-d9d1-4261-b781-d93cbfcb2dc7 kind: Event lastTimestamp: "2026-07-02T13:05:17Z" message: 'Readiness probe failed: Get "https://10.132.0.48:8643/healthz": dial tcp 10.132.0.48:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:05:17Z" name: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4.18be7acb9563e150 namespace: kserve-ci-e2e-test resourceVersion: "22649" uid: 8927c625-90fe-4154-82b8-bc384cb56b61 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-88.ec2.internal source: component: kubelet host: ip-10-0-143-88.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-4a543-predictor-74489d986 namespace: kserve-ci-e2e-test resourceVersion: "21914" uid: 07ae5227-adf1-4821-bccc-13282c98dffa kind: Event lastTimestamp: "2026-07-02T13:03:45Z" message: 'Created pod: raw-sklearn-runtime-4a543-predictor-74489d986-lhhs4' metadata: creationTimestamp: "2026-07-02T13:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:03:45Z" name: raw-sklearn-runtime-4a543-predictor-74489d986.18be7ab62ed6495d namespace: kserve-ci-e2e-test resourceVersion: "21931" uid: 36aa4f0f-7a2d-4bbe-8ad5-72eedd171269 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-4a543-predictor namespace: kserve-ci-e2e-test resourceVersion: "21912" uid: bfdf3988-eb31-45ef-893e-c368ee2190ef kind: Event lastTimestamp: "2026-07-02T13:03:45Z" message: Scaled up replica set raw-sklearn-runtime-4a543-predictor-74489d986 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:03:45Z" name: raw-sklearn-runtime-4a543-predictor.18be7ab623f4457a namespace: kserve-ci-e2e-test resourceVersion: "21915" uid: b7e1a663-d2aa-418f-a323-8d083605d376 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:04:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-4a543-predictor namespace: kserve-ci-e2e-test resourceVersion: "21913" uid: fa3b7404-cce3-496a-8413-8d02e4cea795 kind: Event lastTimestamp: "2026-07-02T13:04:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:04:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:04:30Z" name: raw-sklearn-runtime-4a543-predictor.18be7ab9a2ea17e9 namespace: kserve-ci-e2e-test resourceVersion: "22287" uid: bf5f7e07-0a3e-4d9d-9dcf-b548322b7cc0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:04:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-4a543-predictor namespace: kserve-ci-e2e-test resourceVersion: "21913" uid: fa3b7404-cce3-496a-8413-8d02e4cea795 kind: Event lastTimestamp: "2026-07-02T13:04:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:04:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:04:30Z" name: raw-sklearn-runtime-4a543-predictor.18be7ab9a34d2fc7 namespace: kserve-ci-e2e-test resourceVersion: "22288" uid: 512fb9f9-4d93-4829-b872-2143d12386ae reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:04:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-4a543-predictor namespace: kserve-ci-e2e-test resourceVersion: "22074" uid: fa3b7404-cce3-496a-8413-8d02e4cea795 kind: Event lastTimestamp: "2026-07-02T13:05:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:04:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:05:15Z" name: raw-sklearn-runtime-4a543-predictor.18be7abd22137233 namespace: kserve-ci-e2e-test resourceVersion: "22615" uid: a15467cd-1e6b-4000-9eb8-14c5a2b1a4b6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:04:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-4a543-predictor namespace: kserve-ci-e2e-test resourceVersion: "22074" uid: fa3b7404-cce3-496a-8413-8d02e4cea795 kind: Event lastTimestamp: "2026-07-02T13:05:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:04:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:05:15Z" name: raw-sklearn-runtime-4a543-predictor.18be7abd2276e778 namespace: kserve-ci-e2e-test resourceVersion: "22617" uid: e65a2763-31ad-40c6-824e-c3c5e6d05395 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T13:03:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-4a543 namespace: kserve-ci-e2e-test resourceVersion: "21904" uid: 4b0d18fe-c605-4e83-af5c-355903fa1aaf kind: Event lastTimestamp: "2026-07-02T13:05:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:05:07Z" name: raw-sklearn-runtime-4a543.18be7ab624072652 namespace: kserve-ci-e2e-test resourceVersion: "22564" uid: 36764f51-0923-4b95-a12b-8f2f4d09f3a8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:03:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-4a543 namespace: kserve-ci-e2e-test resourceVersion: "21904" uid: 4b0d18fe-c605-4e83-af5c-355903fa1aaf kind: Event lastTimestamp: "2026-07-02T13:03:45Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-4a543": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-4a543": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:03:45Z" name: raw-sklearn-runtime-4a543.18be7ab62470dd51 namespace: kserve-ci-e2e-test resourceVersion: "21938" uid: 746b1adc-9a78-4428-a80b-43601036220b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-4a543 namespace: kserve-ci-e2e-test resourceVersion: "21906" uid: 4b0d18fe-c605-4e83-af5c-355903fa1aaf kind: Event lastTimestamp: "2026-07-02T13:03:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-4a543": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:03:45Z" name: raw-sklearn-runtime-4a543.18be7ab6300c0989 namespace: kserve-ci-e2e-test resourceVersion: "21940" uid: 34257186-8e37-4dc1-a778-57a71bc1dfff reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:05:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-4a543 namespace: kserve-ci-e2e-test resourceVersion: "22562" uid: 4b0d18fe-c605-4e83-af5c-355903fa1aaf kind: Event lastTimestamp: "2026-07-02T13:05:07Z" message: InferenceService [raw-sklearn-runtime-4a543] is Ready metadata: creationTimestamp: "2026-07-02T13:05:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:05:07Z" name: raw-sklearn-runtime-4a543.18be7ac948466070 namespace: kserve-ci-e2e-test resourceVersion: "22563" uid: 6a57cd1c-d74e-4796-984f-7e8db38b0039 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "22881"