--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:50:57.938501Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-da38c6-predictor-6494f9f9cf-5hbkg namespace: kserve-ci-e2e-test resourceVersion: "19154" uid: dd6a2e8a-d95d-4bba-b88a-3fd625029bb4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-da38c6-predictor-6494f9f9cf-5hbkg to ip-10-0-135-127.ec2.internal metadata: creationTimestamp: "2026-04-17T16:50:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:50:57Z" name: isvc-init-fail-da38c6-predictor-6494f9f9cf-5hbkg.18a733011f90b811 namespace: kserve-ci-e2e-test resourceVersion: "19159" uid: ec96ccd6-656f-447f-865c-6c8262229d28 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f54646664-zzsrx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:50:58Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-da38c6-predictor-6494f9f9cf-5hbkg namespace: kserve-ci-e2e-test resourceVersion: "19157" uid: dd6a2e8a-d95d-4bba-b88a-3fd625029bb4 kind: Event lastTimestamp: "2026-04-17T16:50:58Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-init-fail-da38c6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T16:50:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:50:58Z" name: isvc-init-fail-da38c6-predictor-6494f9f9cf-5hbkg.18a7330128e0ad48 namespace: kserve-ci-e2e-test resourceVersion: "19167" uid: ebd3e102-eb10-4cca-9dfd-7e6f9d1d92af reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:50:58Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-da38c6-predictor-6494f9f9cf-5hbkg namespace: kserve-ci-e2e-test resourceVersion: "19163" uid: dd6a2e8a-d95d-4bba-b88a-3fd625029bb4 kind: Event lastTimestamp: "2026-04-17T16:50:58Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:50:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:50:58Z" name: isvc-init-fail-da38c6-predictor-6494f9f9cf-5hbkg.18a733015c54c96f namespace: kserve-ci-e2e-test resourceVersion: "19183" uid: 33b68732-b198-43ae-985c-746f03b7b180 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:50:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-da38c6-predictor-6494f9f9cf-5hbkg namespace: kserve-ci-e2e-test resourceVersion: "19157" uid: dd6a2e8a-d95d-4bba-b88a-3fd625029bb4 kind: Event lastTimestamp: "2026-04-17T16:51:03Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0cd196d4c53b891914316f18ab5cfa9f85258e057f3687e65332c70bf642d22d" already present on machine metadata: creationTimestamp: "2026-04-17T16:50:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:51:03Z" name: isvc-init-fail-da38c6-predictor-6494f9f9cf-5hbkg.18a733015dde25d7 namespace: kserve-ci-e2e-test resourceVersion: "19228" uid: 5443b4b2-4ada-4d23-ad7f-06db21859a26 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:50:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-da38c6-predictor-6494f9f9cf-5hbkg namespace: kserve-ci-e2e-test resourceVersion: "19157" uid: dd6a2e8a-d95d-4bba-b88a-3fd625029bb4 kind: Event lastTimestamp: "2026-04-17T16:51:03Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-17T16:50:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:51:03Z" name: isvc-init-fail-da38c6-predictor-6494f9f9cf-5hbkg.18a733016189dc1f namespace: kserve-ci-e2e-test resourceVersion: "19231" uid: 1994ea0b-a2ab-4b3b-b988-7373b0eb5e02 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:50:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-da38c6-predictor-6494f9f9cf-5hbkg namespace: kserve-ci-e2e-test resourceVersion: "19157" uid: dd6a2e8a-d95d-4bba-b88a-3fd625029bb4 kind: Event lastTimestamp: "2026-04-17T16:51:03Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-17T16:50:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:51:03Z" name: isvc-init-fail-da38c6-predictor-6494f9f9cf-5hbkg.18a7330161fea0b1 namespace: kserve-ci-e2e-test resourceVersion: "19234" uid: 0a0afc96-d94f-4c0c-983d-b9c7d14d3af2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:51:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-da38c6-predictor-6494f9f9cf-5hbkg namespace: kserve-ci-e2e-test resourceVersion: "19157" uid: dd6a2e8a-d95d-4bba-b88a-3fd625029bb4 kind: Event lastTimestamp: "2026-04-17T16:51:07Z" message: Back-off restarting failed container storage-initializer in pod isvc-init-fail-da38c6-predictor-6494f9f9cf-5hbkg_kserve-ci-e2e-test(dd6a2e8a-d95d-4bba-b88a-3fd625029bb4) metadata: creationTimestamp: "2026-04-17T16:51:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:51:07Z" name: isvc-init-fail-da38c6-predictor-6494f9f9cf-5hbkg.18a733036a01b694 namespace: kserve-ci-e2e-test resourceVersion: "19262" uid: d5d29a0d-f7e7-4b1d-99e0-bd96d380f254 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:50:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-da38c6-predictor-6494f9f9cf namespace: kserve-ci-e2e-test resourceVersion: "19146" uid: 3b9beb57-5103-4808-bfb8-3511b977ce68 kind: Event lastTimestamp: "2026-04-17T16:50:57Z" message: 'Created pod: isvc-init-fail-da38c6-predictor-6494f9f9cf-5hbkg' metadata: creationTimestamp: "2026-04-17T16:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:50:57Z" name: isvc-init-fail-da38c6-predictor-6494f9f9cf.18a733011f25bd53 namespace: kserve-ci-e2e-test resourceVersion: "19156" uid: ae8f40a8-de0d-48d3-89c0-524c8bb7b984 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:50:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-da38c6-predictor namespace: kserve-ci-e2e-test resourceVersion: "19144" uid: 1705e1fb-3acb-4a61-9b72-4b4834e8e623 kind: Event lastTimestamp: "2026-04-17T16:50:57Z" message: Scaled up replica set isvc-init-fail-da38c6-predictor-6494f9f9cf from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:50:57Z" name: isvc-init-fail-da38c6-predictor.18a733011c3aa590 namespace: kserve-ci-e2e-test resourceVersion: "19147" uid: 1429eb7b-33fb-4289-ad8a-f7169181768c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-17T16:50:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-da38c6 namespace: kserve-ci-e2e-test resourceVersion: "19116" uid: 15f7cbfe-8a3a-418b-9e18-da09b39ac862 kind: Event lastTimestamp: "2026-04-17T16:51:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:51:07Z" name: isvc-init-fail-da38c6.18a733011c4514bc namespace: kserve-ci-e2e-test resourceVersion: "19266" uid: 59c84380-b2b0-42c9-a696-d72e7ec03351 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:50:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-da38c6 namespace: kserve-ci-e2e-test resourceVersion: "19116" uid: 15f7cbfe-8a3a-418b-9e18-da09b39ac862 kind: Event lastTimestamp: "2026-04-17T16:51:07Z" message: 'Failed to update status for InferenceService "isvc-init-fail-da38c6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-da38c6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:51:07Z" name: isvc-init-fail-da38c6.18a733011cbe8587 namespace: kserve-ci-e2e-test resourceVersion: "19267" uid: f2a1a5b0-d5dc-4433-bc6b-08cd9168b03e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:50:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-da38c6 namespace: kserve-ci-e2e-test resourceVersion: "19116" uid: 15f7cbfe-8a3a-418b-9e18-da09b39ac862 kind: Event lastTimestamp: "2026-04-17T16:51:07Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-da38c6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:51:07Z" name: isvc-init-fail-da38c6.18a733011cbecb17 namespace: kserve-ci-e2e-test resourceVersion: "19270" uid: a5db2ec2-2d11-44bf-b764-8a7846aa9dbf reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:45:24.053750Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9 namespace: kserve-ci-e2e-test resourceVersion: "16392" uid: f8b87406-cbce-4e83-8693-c171eb8196b5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9 to ip-10-0-135-127.ec2.internal metadata: creationTimestamp: "2026-04-17T16:45:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:45:24Z" name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9.18a732b3627bb588 namespace: kserve-ci-e2e-test resourceVersion: "16396" uid: b9f1aa44-9d25-422f-a369-37825647b758 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f54646664-zzsrx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:24Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9 namespace: kserve-ci-e2e-test resourceVersion: "16401" uid: f8b87406-cbce-4e83-8693-c171eb8196b5 kind: Event lastTimestamp: "2026-04-17T16:45:24Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:45:24Z" name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9.18a732b37b7e0810 namespace: kserve-ci-e2e-test resourceVersion: "16406" uid: d075c80c-390d-482f-a3cd-0345a4a8b57e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9 namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: f8b87406-cbce-4e83-8693-c171eb8196b5 kind: Event lastTimestamp: "2026-04-17T16:45:24Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0cd196d4c53b891914316f18ab5cfa9f85258e057f3687e65332c70bf642d22d" already present on machine metadata: creationTimestamp: "2026-04-17T16:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:24Z" name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9.18a732b37d0dc48e namespace: kserve-ci-e2e-test resourceVersion: "16408" uid: 51f65ec8-8225-45c2-a43f-ba51d7da9ca7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9 namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: f8b87406-cbce-4e83-8693-c171eb8196b5 kind: Event lastTimestamp: "2026-04-17T16:45:24Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-17T16:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:24Z" name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9.18a732b37fb57cb3 namespace: kserve-ci-e2e-test resourceVersion: "16409" uid: 01f07785-efe4-4888-8f1b-87fe54f7110e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9 namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: f8b87406-cbce-4e83-8693-c171eb8196b5 kind: Event lastTimestamp: "2026-04-17T16:45:24Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-17T16:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:24Z" name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9.18a732b38027ed6a namespace: kserve-ci-e2e-test resourceVersion: "16410" uid: ccbe361b-2ab5-4443-9f80-7c0680ad2d07 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9 namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: f8b87406-cbce-4e83-8693-c171eb8196b5 kind: Event lastTimestamp: "2026-04-17T16:45:28Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T16:45:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:28Z" name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9.18a732b4777dadf4 namespace: kserve-ci-e2e-test resourceVersion: "16446" uid: 06da76a2-0338-4615-8d43-2cdd5c277d0c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9 namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: f8b87406-cbce-4e83-8693-c171eb8196b5 kind: Event lastTimestamp: "2026-04-17T16:45:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:45:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:28Z" name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9.18a732b47aa05501 namespace: kserve-ci-e2e-test resourceVersion: "16451" uid: 0d995b82-2a96-4f55-9273-04f05667167b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9 namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: f8b87406-cbce-4e83-8693-c171eb8196b5 kind: Event lastTimestamp: "2026-04-17T16:45:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:45:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:28Z" name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9.18a732b47b1438a8 namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: 2357d7fb-5c19-4917-87c2-946a7f21c882 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9 namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: f8b87406-cbce-4e83-8693-c171eb8196b5 kind: Event lastTimestamp: "2026-04-17T16:45:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T16:45:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:28Z" name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9.18a732b47b21d5c4 namespace: kserve-ci-e2e-test resourceVersion: "16453" uid: 1da7cabe-2a4f-410b-8d9c-22e3ca0b7b8c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9 namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: f8b87406-cbce-4e83-8693-c171eb8196b5 kind: Event lastTimestamp: "2026-04-17T16:45:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:45:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:28Z" name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9.18a732b47dd83f4d namespace: kserve-ci-e2e-test resourceVersion: "16456" uid: 29b94b51-c691-4d3b-b51b-125d23742b3b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9 namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: f8b87406-cbce-4e83-8693-c171eb8196b5 kind: Event lastTimestamp: "2026-04-17T16:45:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:45:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:28Z" name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9.18a732b47e49d92a namespace: kserve-ci-e2e-test resourceVersion: "16457" uid: ac323997-8d35-4ded-a2ff-e0ab910f36f4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9 namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: f8b87406-cbce-4e83-8693-c171eb8196b5 kind: Event lastTimestamp: "2026-04-17T16:45:28Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:63b01855d4f4d9cc9a109698dbd7d7889d3c2d40a32577f5946d367ab2c8c321" already present on machine metadata: creationTimestamp: "2026-04-17T16:45:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:28Z" name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9.18a732b47e56ca42 namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: 142ab79d-81f5-44db-a98f-06be30cbbb36 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9 namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: f8b87406-cbce-4e83-8693-c171eb8196b5 kind: Event lastTimestamp: "2026-04-17T16:45:28Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-17T16:45:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:28Z" name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9.18a732b48110dd37 namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: f28dff73-23a4-418f-b11c-23e8bf60516f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9 namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: f8b87406-cbce-4e83-8693-c171eb8196b5 kind: Event lastTimestamp: "2026-04-17T16:45:28Z" message: Started container agent metadata: creationTimestamp: "2026-04-17T16:45:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:28Z" name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9.18a732b48185c505 namespace: kserve-ci-e2e-test resourceVersion: "16460" uid: 6e847c1b-bd64-405b-b2dc-f3ac5b36ddf8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-17T16:45:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9 namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: f8b87406-cbce-4e83-8693-c171eb8196b5 kind: Event lastTimestamp: "2026-04-17T16:47:06Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:45:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:47:06Z" name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9.18a732b4ef34aadb namespace: kserve-ci-e2e-test resourceVersion: "17296" uid: 77ae822d-b4f6-4ff7-9d53-990bfbbf9c98 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-17T16:45:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9 namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: f8b87406-cbce-4e83-8693-c171eb8196b5 kind: Event lastTimestamp: "2026-04-17T16:47:06Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T16:45:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:47:06Z" name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9.18a732b4ef3e9b24 namespace: kserve-ci-e2e-test resourceVersion: "17297" uid: 785a0e54-a913-422e-878c-bfc8eb449ab2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:46:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9 namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: f8b87406-cbce-4e83-8693-c171eb8196b5 kind: Event lastTimestamp: "2026-04-17T16:46:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T16:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:46:49Z" name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9.18a732c7397d00c4 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 0c674f42-bf5f-4324-a9ee-7f36c40c7c35 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:46:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9 namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: f8b87406-cbce-4e83-8693-c171eb8196b5 kind: Event lastTimestamp: "2026-04-17T16:46:49Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-17T16:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:46:49Z" name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9.18a732c7397d4a8e namespace: kserve-ci-e2e-test resourceVersion: "17095" uid: 08480e7b-96a2-4d1e-b6f3-e933f26f924e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:46:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9 namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: f8b87406-cbce-4e83-8693-c171eb8196b5 kind: Event lastTimestamp: "2026-04-17T16:46:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:46:49Z" name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9.18a732c7397d8fd2 namespace: kserve-ci-e2e-test resourceVersion: "17096" uid: 96d66d2d-7f7a-4ff3-877f-ab6a23a1201e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T16:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9 namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: f8b87406-cbce-4e83-8693-c171eb8196b5 kind: Event lastTimestamp: "2026-04-17T16:47:11Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:47:11Z" name: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9.18a732c7cb55f170 namespace: kserve-ci-e2e-test resourceVersion: "17330" uid: 841f6e0f-143b-4b8b-aa64-6ed0c92da8b7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-cd35e-predictor-8448c47648 namespace: kserve-ci-e2e-test resourceVersion: "16382" uid: 29afe678-6593-4dc4-b58b-be058ac08481 kind: Event lastTimestamp: "2026-04-17T16:45:24Z" message: 'Created pod: isvc-logger-raw-cd35e-predictor-8448c47648-kjwr9' metadata: creationTimestamp: "2026-04-17T16:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:45:24Z" name: isvc-logger-raw-cd35e-predictor-8448c47648.18a732b362111a27 namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: 79ac79a2-b96c-496a-bde3-668524e06a42 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-cd35e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: 08a380ee-e69f-42f7-bf3a-7a9a0bb63d92 kind: Event lastTimestamp: "2026-04-17T16:45:23Z" message: Scaled up replica set isvc-logger-raw-cd35e-predictor-8448c47648 from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:45:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:45:23Z" name: isvc-logger-raw-cd35e-predictor.18a732b35eea4f00 namespace: kserve-ci-e2e-test resourceVersion: "16384" uid: 2708f68c-e87f-4ec0-a636-9a8535461105 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:45:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-cd35e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: 418263a0-ca5f-4965-9a68-bc088ebb6686 kind: Event lastTimestamp: "2026-04-17T16:46:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:46:09Z" name: isvc-logger-raw-cd35e-predictor.18a732b6de3e741d namespace: kserve-ci-e2e-test resourceVersion: "16766" uid: 0fd0c9d7-8736-4baf-9d68-c46ca7b993b7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:45:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-cd35e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: 418263a0-ca5f-4965-9a68-bc088ebb6686 kind: Event lastTimestamp: "2026-04-17T16:46:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:46:09Z" name: isvc-logger-raw-cd35e-predictor.18a732b6dea180b8 namespace: kserve-ci-e2e-test resourceVersion: "16767" uid: c260eda2-819c-4b29-96b5-9c15290c8a7d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:45:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-cd35e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16541" uid: 418263a0-ca5f-4965-9a68-bc088ebb6686 kind: Event lastTimestamp: "2026-04-17T16:46:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:45:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:46:39Z" name: isvc-logger-raw-cd35e-predictor.18a732ba5d4c5edd namespace: kserve-ci-e2e-test resourceVersion: "16988" uid: 9f95da0f-c96f-4697-9b4d-150a98870e58 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:45:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-cd35e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16541" uid: 418263a0-ca5f-4965-9a68-bc088ebb6686 kind: Event lastTimestamp: "2026-04-17T16:46:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:45:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:46:39Z" name: isvc-logger-raw-cd35e-predictor.18a732ba5db22b68 namespace: kserve-ci-e2e-test resourceVersion: "16989" uid: 8dd24b9f-b710-42e6-ada2-e06dd949c677 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-17T16:45:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-cd35e namespace: kserve-ci-e2e-test resourceVersion: "16371" uid: fa3bee45-985b-42d7-abfb-b8fee2f837e8 kind: Event lastTimestamp: "2026-04-17T16:46:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:45:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:46:36Z" name: isvc-logger-raw-cd35e.18a732b35ec7194e namespace: kserve-ci-e2e-test resourceVersion: "16970" uid: 6d2f7bf2-7170-49a1-83ce-fe449ab06b7c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-cd35e namespace: kserve-ci-e2e-test resourceVersion: "16371" uid: fa3bee45-985b-42d7-abfb-b8fee2f837e8 kind: Event lastTimestamp: "2026-04-17T16:45:23Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-cd35e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-cd35e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:45:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:45:23Z" name: isvc-logger-raw-cd35e.18a732b35f1e31a7 namespace: kserve-ci-e2e-test resourceVersion: "16386" uid: c2042f87-2370-411b-af58-53eaf91fcc3c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:46:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-cd35e namespace: kserve-ci-e2e-test resourceVersion: "16968" uid: fa3bee45-985b-42d7-abfb-b8fee2f837e8 kind: Event lastTimestamp: "2026-04-17T16:46:36Z" message: InferenceService [isvc-logger-raw-cd35e] is Ready metadata: creationTimestamp: "2026-04-17T16:46:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:46:36Z" name: isvc-logger-raw-cd35e.18a732c4533a051b namespace: kserve-ci-e2e-test resourceVersion: "16969" uid: c1096a8f-1995-4818-b227-37ac3630c078 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:49:19.607143Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc namespace: kserve-ci-e2e-test resourceVersion: "18282" uid: e8fc05a8-d184-4066-96e9-593eeddd8107 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-c62bcc-predictor-75f99866db-zhgfc to ip-10-0-135-127.ec2.internal metadata: creationTimestamp: "2026-04-17T16:49:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:49:19Z" name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc.18a732ea3a8f403b namespace: kserve-ci-e2e-test resourceVersion: "18286" uid: 529daf0a-d95e-4ed2-86a4-cc1464773872 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f54646664-zzsrx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:49:20Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc namespace: kserve-ci-e2e-test resourceVersion: "18290" uid: e8fc05a8-d184-4066-96e9-593eeddd8107 kind: Event lastTimestamp: "2026-04-17T16:49:20Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:49:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:49:20Z" name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc.18a732ea537d5c77 namespace: kserve-ci-e2e-test resourceVersion: "18298" uid: 6c1a8bd2-05c0-471c-bff6-469d4b3e9c74 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:49:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc namespace: kserve-ci-e2e-test resourceVersion: "18284" uid: e8fc05a8-d184-4066-96e9-593eeddd8107 kind: Event lastTimestamp: "2026-04-17T16:49:20Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0cd196d4c53b891914316f18ab5cfa9f85258e057f3687e65332c70bf642d22d" already present on machine metadata: creationTimestamp: "2026-04-17T16:49:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:49:20Z" name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc.18a732ea54df6cfe namespace: kserve-ci-e2e-test resourceVersion: "18300" uid: 668c8ab0-c508-46cf-b740-224fa6f33e32 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:49:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc namespace: kserve-ci-e2e-test resourceVersion: "18284" uid: e8fc05a8-d184-4066-96e9-593eeddd8107 kind: Event lastTimestamp: "2026-04-17T16:49:20Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-17T16:49:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:49:20Z" name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc.18a732ea57c17656 namespace: kserve-ci-e2e-test resourceVersion: "18301" uid: 3770701c-f306-4b3a-b030-c2c6fd37b59f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:49:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc namespace: kserve-ci-e2e-test resourceVersion: "18284" uid: e8fc05a8-d184-4066-96e9-593eeddd8107 kind: Event lastTimestamp: "2026-04-17T16:49:20Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-17T16:49:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:49:20Z" name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc.18a732ea58326f3a namespace: kserve-ci-e2e-test resourceVersion: "18302" uid: 48c64e8a-aa31-43de-98da-c2de2de1a1be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:49:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc namespace: kserve-ci-e2e-test resourceVersion: "18284" uid: e8fc05a8-d184-4066-96e9-593eeddd8107 kind: Event lastTimestamp: "2026-04-17T16:49:24Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T16:49:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:49:24Z" name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc.18a732eb5a8a8326 namespace: kserve-ci-e2e-test resourceVersion: "18340" uid: 8bbff4a8-e909-4ec8-b8eb-8a85568da378 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:49:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc namespace: kserve-ci-e2e-test resourceVersion: "18284" uid: e8fc05a8-d184-4066-96e9-593eeddd8107 kind: Event lastTimestamp: "2026-04-17T16:49:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:49:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:49:24Z" name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc.18a732eb5d4ca74c namespace: kserve-ci-e2e-test resourceVersion: "18342" uid: fa817b39-0618-4b17-9f49-8d4b2a72d6d0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:49:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc namespace: kserve-ci-e2e-test resourceVersion: "18284" uid: e8fc05a8-d184-4066-96e9-593eeddd8107 kind: Event lastTimestamp: "2026-04-17T16:49:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:49:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:49:24Z" name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc.18a732eb5dbf60e2 namespace: kserve-ci-e2e-test resourceVersion: "18345" uid: bdcad218-10df-49de-980e-4f0c09c7b4c1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:49:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc namespace: kserve-ci-e2e-test resourceVersion: "18284" uid: e8fc05a8-d184-4066-96e9-593eeddd8107 kind: Event lastTimestamp: "2026-04-17T16:49:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T16:49:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:49:24Z" name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc.18a732eb5dcccc2b namespace: kserve-ci-e2e-test resourceVersion: "18346" uid: fc2a89fc-8a73-48a1-a2b8-0e1f062fbd61 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:49:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc namespace: kserve-ci-e2e-test resourceVersion: "18284" uid: e8fc05a8-d184-4066-96e9-593eeddd8107 kind: Event lastTimestamp: "2026-04-17T16:49:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:49:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:49:24Z" name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc.18a732eb607fc1ea namespace: kserve-ci-e2e-test resourceVersion: "18348" uid: 9f55389c-fcf6-4351-be5e-a9eddaf34ca2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:49:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc namespace: kserve-ci-e2e-test resourceVersion: "18284" uid: e8fc05a8-d184-4066-96e9-593eeddd8107 kind: Event lastTimestamp: "2026-04-17T16:49:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:49:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:49:24Z" name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc.18a732eb60f16d76 namespace: kserve-ci-e2e-test resourceVersion: "18349" uid: 58036dfc-c925-49a9-a1fc-00a947d3b284 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-17T16:49:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc namespace: kserve-ci-e2e-test resourceVersion: "18284" uid: e8fc05a8-d184-4066-96e9-593eeddd8107 kind: Event lastTimestamp: "2026-04-17T16:50:22Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:50:22Z" name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc.18a732ebd2419604 namespace: kserve-ci-e2e-test resourceVersion: "18764" uid: 08adbe89-933e-4647-8d2b-0a1344b4c35a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:50:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc namespace: kserve-ci-e2e-test resourceVersion: "18284" uid: e8fc05a8-d184-4066-96e9-593eeddd8107 kind: Event lastTimestamp: "2026-04-17T16:50:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T16:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:50:57Z" name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc.18a733011aa2b345 namespace: kserve-ci-e2e-test resourceVersion: "19141" uid: 493f55cf-f642-47b5-b349-dbf1c36683b1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:50:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc namespace: kserve-ci-e2e-test resourceVersion: "18284" uid: e8fc05a8-d184-4066-96e9-593eeddd8107 kind: Event lastTimestamp: "2026-04-17T16:50:57Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:50:57Z" name: isvc-primary-c62bcc-predictor-75f99866db-zhgfc.18a733011aa6b385 namespace: kserve-ci-e2e-test resourceVersion: "19139" uid: 08c3b05c-b87c-46ca-aa24-1a01b101a82a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:49:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-c62bcc-predictor-75f99866db namespace: kserve-ci-e2e-test resourceVersion: "18273" uid: 2d94802e-6481-441f-a9c2-7da252c0698c kind: Event lastTimestamp: "2026-04-17T16:49:19Z" message: 'Created pod: isvc-primary-c62bcc-predictor-75f99866db-zhgfc' metadata: creationTimestamp: "2026-04-17T16:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:49:19Z" name: isvc-primary-c62bcc-predictor-75f99866db.18a732ea3a238d7b namespace: kserve-ci-e2e-test resourceVersion: "18283" uid: 258ff71d-b94c-4b7a-9ebb-4f025e007a2f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:49:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-c62bcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18271" uid: 92190bee-7d3a-498a-8d70-67dab6648aa9 kind: Event lastTimestamp: "2026-04-17T16:49:19Z" message: Scaled up replica set isvc-primary-c62bcc-predictor-75f99866db from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:49:19Z" name: isvc-primary-c62bcc-predictor.18a732ea34c90f10 namespace: kserve-ci-e2e-test resourceVersion: "18275" uid: 2f3e82ca-438e-4862-a964-8fb6dab0aba1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:49:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-c62bcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18272" uid: 02cab974-84c2-4ed2-b86e-18fcf4e02cb0 kind: Event lastTimestamp: "2026-04-17T16:49:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:49:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:49:49Z" name: isvc-primary-c62bcc-predictor.18a732edb34809ae namespace: kserve-ci-e2e-test resourceVersion: "18536" uid: dd996d6a-353b-4cf6-8c59-10aea19d6f3c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:49:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-c62bcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18272" uid: 02cab974-84c2-4ed2-b86e-18fcf4e02cb0 kind: Event lastTimestamp: "2026-04-17T16:49:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:49:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:49:49Z" name: isvc-primary-c62bcc-predictor.18a732edb3a6b3c2 namespace: kserve-ci-e2e-test resourceVersion: "18537" uid: 71cdbe2e-6d6d-47bc-8533-833d37ada7d2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T16:50:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-c62bcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18429" uid: 02cab974-84c2-4ed2-b86e-18fcf4e02cb0 kind: Event lastTimestamp: "2026-04-17T16:50:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:50:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:50:49Z" name: isvc-primary-c62bcc-predictor.18a732f4b1beafeb namespace: kserve-ci-e2e-test resourceVersion: "19026" uid: ef64257f-8df3-46a2-a5e2-a0bfa4e9dd96 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T16:50:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-c62bcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18429" uid: 02cab974-84c2-4ed2-b86e-18fcf4e02cb0 kind: Event lastTimestamp: "2026-04-17T16:50:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:50:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:50:49Z" name: isvc-primary-c62bcc-predictor.18a732f4b208d6ba namespace: kserve-ci-e2e-test resourceVersion: "19027" uid: 5d7cadab-d2a0-4f6d-a8a6-1026f5b15dd6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-17T16:49:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-c62bcc namespace: kserve-ci-e2e-test resourceVersion: "18241" uid: 693c2794-202d-4ba7-87f3-17d4f774271a kind: Event lastTimestamp: "2026-04-17T16:50:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:50:32Z" name: isvc-primary-c62bcc.18a732ea34a3d563 namespace: kserve-ci-e2e-test resourceVersion: "18846" uid: e6c8e9ae-34cc-4811-9e9c-3e7d767b5d25 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:49:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-c62bcc namespace: kserve-ci-e2e-test resourceVersion: "18241" uid: 693c2794-202d-4ba7-87f3-17d4f774271a kind: Event lastTimestamp: "2026-04-17T16:49:19Z" message: 'Failed to update status for InferenceService "isvc-primary-c62bcc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-c62bcc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:49:19Z" name: isvc-primary-c62bcc.18a732ea3520097a namespace: kserve-ci-e2e-test resourceVersion: "18277" uid: d60eb80c-501b-4082-933e-de31d8f0b3b1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:49:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-c62bcc namespace: kserve-ci-e2e-test resourceVersion: "18241" uid: 693c2794-202d-4ba7-87f3-17d4f774271a kind: Event lastTimestamp: "2026-04-17T16:49:19Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-c62bcc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:49:19Z" name: isvc-primary-c62bcc.18a732ea35205ffc namespace: kserve-ci-e2e-test resourceVersion: "18278" uid: 57e67ff8-8098-4c19-9eb1-2cdf3331fe1a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:50:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-c62bcc namespace: kserve-ci-e2e-test resourceVersion: "18844" uid: 693c2794-202d-4ba7-87f3-17d4f774271a kind: Event lastTimestamp: "2026-04-17T16:50:32Z" message: InferenceService [isvc-primary-c62bcc] is Ready metadata: creationTimestamp: "2026-04-17T16:50:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:50:32Z" name: isvc-primary-c62bcc.18a732fb35620092 namespace: kserve-ci-e2e-test resourceVersion: "18845" uid: b43a86b2-fc6a-4612-ae48-f9582fccb4fd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:39:58.281292Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2 namespace: kserve-ci-e2e-test resourceVersion: "13365" uid: 6947aac7-9c49-4c08-9cc4-060fa22367f2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2 to ip-10-0-135-127.ec2.internal metadata: creationTimestamp: "2026-04-17T16:39:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:39:58Z" name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2.18a7326788ee90c1 namespace: kserve-ci-e2e-test resourceVersion: "13370" uid: 506f82dc-d107-473f-a144-601de377f09e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f54646664-zzsrx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:39:58Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2 namespace: kserve-ci-e2e-test resourceVersion: "13377" uid: 6947aac7-9c49-4c08-9cc4-060fa22367f2 kind: Event lastTimestamp: "2026-04-17T16:39:58Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:39:58Z" name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2.18a73267a2376ce3 namespace: kserve-ci-e2e-test resourceVersion: "13388" uid: cbb3d247-613c-496e-892d-b7161e0b7023 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:39:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2 namespace: kserve-ci-e2e-test resourceVersion: "13367" uid: 6947aac7-9c49-4c08-9cc4-060fa22367f2 kind: Event lastTimestamp: "2026-04-17T16:39:58Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0cd196d4c53b891914316f18ab5cfa9f85258e057f3687e65332c70bf642d22d" metadata: creationTimestamp: "2026-04-17T16:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:39:58Z" name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2.18a73267a3f3b8c2 namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: e2ccefbe-0142-43a6-9064-8bfd11ff2542 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:40:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2 namespace: kserve-ci-e2e-test resourceVersion: "13367" uid: 6947aac7-9c49-4c08-9cc4-060fa22367f2 kind: Event lastTimestamp: "2026-04-17T16:40:03Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0cd196d4c53b891914316f18ab5cfa9f85258e057f3687e65332c70bf642d22d" in 4.361s (4.361s including waiting). Image size: 301288360 bytes.' metadata: creationTimestamp: "2026-04-17T16:40:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:40:03Z" name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2.18a73268a7eeddd6 namespace: kserve-ci-e2e-test resourceVersion: "13427" uid: a97acc50-da2a-497f-9f08-a27ae516503e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:40:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2 namespace: kserve-ci-e2e-test resourceVersion: "13367" uid: 6947aac7-9c49-4c08-9cc4-060fa22367f2 kind: Event lastTimestamp: "2026-04-17T16:40:03Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-17T16:40:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:40:03Z" name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2.18a73268ab8372f9 namespace: kserve-ci-e2e-test resourceVersion: "13428" uid: 0c747a76-68a6-4d65-8080-6ac0fc513ebd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:40:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2 namespace: kserve-ci-e2e-test resourceVersion: "13367" uid: 6947aac7-9c49-4c08-9cc4-060fa22367f2 kind: Event lastTimestamp: "2026-04-17T16:40:03Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-17T16:40:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:40:03Z" name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2.18a73268ac04ca6f namespace: kserve-ci-e2e-test resourceVersion: "13429" uid: c3d738d6-78e7-4587-bfab-97c337cc63a8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:40:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2 namespace: kserve-ci-e2e-test resourceVersion: "13367" uid: 6947aac7-9c49-4c08-9cc4-060fa22367f2 kind: Event lastTimestamp: "2026-04-17T16:40:06Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" metadata: creationTimestamp: "2026-04-17T16:40:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:40:06Z" name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2.18a732697bf20a20 namespace: kserve-ci-e2e-test resourceVersion: "13461" uid: bcc7fedd-e81f-4c9a-95f6-a1a90929858e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:40:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2 namespace: kserve-ci-e2e-test resourceVersion: "13367" uid: 6947aac7-9c49-4c08-9cc4-060fa22367f2 kind: Event lastTimestamp: "2026-04-17T16:40:19Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" in 12.981s (12.981s including waiting). Image size: 1560612266 bytes.' metadata: creationTimestamp: "2026-04-17T16:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:40:19Z" name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2.18a7326c81b4f393 namespace: kserve-ci-e2e-test resourceVersion: "13558" uid: 63f78159-be97-48d8-8bc0-2fb476bab36f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:40:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2 namespace: kserve-ci-e2e-test resourceVersion: "13367" uid: 6947aac7-9c49-4c08-9cc4-060fa22367f2 kind: Event lastTimestamp: "2026-04-17T16:40:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:40:19Z" name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2.18a7326c8582456d namespace: kserve-ci-e2e-test resourceVersion: "13559" uid: 0a50f4ab-7e91-40e5-8e26-4ecf08e9336c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:40:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2 namespace: kserve-ci-e2e-test resourceVersion: "13367" uid: 6947aac7-9c49-4c08-9cc4-060fa22367f2 kind: Event lastTimestamp: "2026-04-17T16:40:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:40:19Z" name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2.18a7326c85fad0b1 namespace: kserve-ci-e2e-test resourceVersion: "13560" uid: f2549f62-dca5-432d-8597-685184a29314 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:40:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2 namespace: kserve-ci-e2e-test resourceVersion: "13367" uid: 6947aac7-9c49-4c08-9cc4-060fa22367f2 kind: Event lastTimestamp: "2026-04-17T16:40:19Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-17T16:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:40:19Z" name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2.18a7326c8607ca9c namespace: kserve-ci-e2e-test resourceVersion: "13561" uid: 7198764b-4b15-45b3-88a8-537de758e1ce reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:40:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2 namespace: kserve-ci-e2e-test resourceVersion: "13367" uid: 6947aac7-9c49-4c08-9cc4-060fa22367f2 kind: Event lastTimestamp: "2026-04-17T16:40:22Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.595s (2.595s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-17T16:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:40:22Z" name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2.18a7326d20baa3d4 namespace: kserve-ci-e2e-test resourceVersion: "13578" uid: 55b00a89-7edd-47c1-aac8-85270393aa62 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:40:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2 namespace: kserve-ci-e2e-test resourceVersion: "13367" uid: 6947aac7-9c49-4c08-9cc4-060fa22367f2 kind: Event lastTimestamp: "2026-04-17T16:40:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:40:22Z" name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2.18a7326d243a96b3 namespace: kserve-ci-e2e-test resourceVersion: "13579" uid: 4f8a2df1-8da4-40a4-856a-a37911bd9f30 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:40:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2 namespace: kserve-ci-e2e-test resourceVersion: "13367" uid: 6947aac7-9c49-4c08-9cc4-060fa22367f2 kind: Event lastTimestamp: "2026-04-17T16:40:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:40:22Z" name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2.18a7326d24adfab8 namespace: kserve-ci-e2e-test resourceVersion: "13580" uid: c27708a7-951b-4164-9607-0f62fe1d6308 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:40:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2 namespace: kserve-ci-e2e-test resourceVersion: "13367" uid: 6947aac7-9c49-4c08-9cc4-060fa22367f2 kind: Event lastTimestamp: "2026-04-17T16:40:22Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:63b01855d4f4d9cc9a109698dbd7d7889d3c2d40a32577f5946d367ab2c8c321" metadata: creationTimestamp: "2026-04-17T16:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:40:22Z" name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2.18a7326d24ba1c8d namespace: kserve-ci-e2e-test resourceVersion: "13581" uid: 293bafa2-7917-4100-a175-b6dfa63bf8dc reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:40:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2 namespace: kserve-ci-e2e-test resourceVersion: "13367" uid: 6947aac7-9c49-4c08-9cc4-060fa22367f2 kind: Event lastTimestamp: "2026-04-17T16:40:25Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:63b01855d4f4d9cc9a109698dbd7d7889d3c2d40a32577f5946d367ab2c8c321" in 2.794s (2.794s including waiting). Image size: 237897293 bytes.' metadata: creationTimestamp: "2026-04-17T16:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:40:25Z" name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2.18a7326dcb51f07a namespace: kserve-ci-e2e-test resourceVersion: "13604" uid: b2d461c1-890b-40a2-b1a9-238323ecb5c7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:40:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2 namespace: kserve-ci-e2e-test resourceVersion: "13367" uid: 6947aac7-9c49-4c08-9cc4-060fa22367f2 kind: Event lastTimestamp: "2026-04-17T16:40:25Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-17T16:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:40:25Z" name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2.18a7326dce448404 namespace: kserve-ci-e2e-test resourceVersion: "13606" uid: a9b75808-ec14-4f7f-bd47-3c85198bcda9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:40:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2 namespace: kserve-ci-e2e-test resourceVersion: "13367" uid: 6947aac7-9c49-4c08-9cc4-060fa22367f2 kind: Event lastTimestamp: "2026-04-17T16:40:25Z" message: Started container agent metadata: creationTimestamp: "2026-04-17T16:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:40:25Z" name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2.18a7326dceb78af9 namespace: kserve-ci-e2e-test resourceVersion: "13607" uid: 2a438043-1ba0-4ec9-8108-3e12536cd806 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-17T16:40:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2 namespace: kserve-ci-e2e-test resourceVersion: "13367" uid: 6947aac7-9c49-4c08-9cc4-060fa22367f2 kind: Event lastTimestamp: "2026-04-17T16:41:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:58Z" name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2.18a7326ded2aac4f namespace: kserve-ci-e2e-test resourceVersion: "14428" uid: 2e79e5ca-5118-4730-8e8d-6027a5e45171 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-17T16:40:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2 namespace: kserve-ci-e2e-test resourceVersion: "13367" uid: 6947aac7-9c49-4c08-9cc4-060fa22367f2 kind: Event lastTimestamp: "2026-04-17T16:41:58Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T16:40:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:58Z" name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2.18a7326e28e5fbba namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: 95fb49a9-8919-4064-96d1-89c04ff1b7ce reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2 namespace: kserve-ci-e2e-test resourceVersion: "13367" uid: 6947aac7-9c49-4c08-9cc4-060fa22367f2 kind: Event lastTimestamp: "2026-04-17T16:41:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T16:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:43Z" name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2.18a7327ffeb28c6c namespace: kserve-ci-e2e-test resourceVersion: "14212" uid: 63de5b6a-a701-40ac-baa9-4f130534b054 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2 namespace: kserve-ci-e2e-test resourceVersion: "13367" uid: 6947aac7-9c49-4c08-9cc4-060fa22367f2 kind: Event lastTimestamp: "2026-04-17T16:41:43Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-17T16:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:43Z" name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2.18a7327ffeb2be71 namespace: kserve-ci-e2e-test resourceVersion: "14213" uid: 151aadf4-d777-4cb7-b53e-5cc17d4e0038 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2 namespace: kserve-ci-e2e-test resourceVersion: "13367" uid: 6947aac7-9c49-4c08-9cc4-060fa22367f2 kind: Event lastTimestamp: "2026-04-17T16:41:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:43Z" name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2.18a7327ffeb2f941 namespace: kserve-ci-e2e-test resourceVersion: "14215" uid: 87792dc8-dd17-4c44-b86c-1643b955fa10 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T16:41:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2 namespace: kserve-ci-e2e-test resourceVersion: "13367" uid: 6947aac7-9c49-4c08-9cc4-060fa22367f2 kind: Event lastTimestamp: "2026-04-17T16:42:02Z" message: 'Readiness probe failed: Get "https://10.134.0.20:8643/healthz": dial tcp 10.134.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:41:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:02Z" name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2.18a7328105162d79 namespace: kserve-ci-e2e-test resourceVersion: "14456" uid: 262ade10-238c-4229-a73d-70953be5e8d8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:39:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949 namespace: kserve-ci-e2e-test resourceVersion: "13358" uid: 95e664fe-cd2b-4534-bce7-c3fa1bc20f50 kind: Event lastTimestamp: "2026-04-17T16:39:58Z" message: 'Created pod: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949-dzdr2' metadata: creationTimestamp: "2026-04-17T16:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:39:58Z" name: isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949.18a73267887737f6 namespace: kserve-ci-e2e-test resourceVersion: "13366" uid: 11482caa-7959-40af-87f4-d5d4faecb62b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:39:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-2a85d-predictor namespace: kserve-ci-e2e-test resourceVersion: "13357" uid: 466b8354-f184-4f9a-9e01-f97864a2168f kind: Event lastTimestamp: "2026-04-17T16:39:58Z" message: Scaled up replica set isvc-raw-sklearn-batcher-2a85d-predictor-5dfd9b949 from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:39:58Z" name: isvc-raw-sklearn-batcher-2a85d-predictor.18a73267834708f2 namespace: kserve-ci-e2e-test resourceVersion: "13359" uid: 039b3a5b-35e4-4fde-84db-042526344430 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:40:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-2a85d-predictor namespace: kserve-ci-e2e-test resourceVersion: "13369" uid: caa72829-2bc7-4310-a2e8-bf379ef5a485 kind: Event lastTimestamp: "2026-04-17T16:40:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:40:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:40:43Z" name: isvc-raw-sklearn-batcher-2a85d-predictor.18a7326b090391b8 namespace: kserve-ci-e2e-test resourceVersion: "13742" uid: 9ad3d22c-d892-44b8-b270-823f6e74dc9b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:40:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-2a85d-predictor namespace: kserve-ci-e2e-test resourceVersion: "13369" uid: caa72829-2bc7-4310-a2e8-bf379ef5a485 kind: Event lastTimestamp: "2026-04-17T16:40:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:40:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:40:43Z" name: isvc-raw-sklearn-batcher-2a85d-predictor.18a7326b097402cb namespace: kserve-ci-e2e-test resourceVersion: "13743" uid: 95d5898f-d193-444d-870f-6babf74d9682 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:40:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-2a85d-predictor namespace: kserve-ci-e2e-test resourceVersion: "13512" uid: caa72829-2bc7-4310-a2e8-bf379ef5a485 kind: Event lastTimestamp: "2026-04-17T16:41:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:41:28Z" name: isvc-raw-sklearn-batcher-2a85d-predictor.18a7327586443b38 namespace: kserve-ci-e2e-test resourceVersion: "14065" uid: d3d39133-ff4c-4272-98dc-c5dd7fb98216 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:40:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-2a85d-predictor namespace: kserve-ci-e2e-test resourceVersion: "13512" uid: caa72829-2bc7-4310-a2e8-bf379ef5a485 kind: Event lastTimestamp: "2026-04-17T16:41:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:41:28Z" name: isvc-raw-sklearn-batcher-2a85d-predictor.18a7327586af9fc3 namespace: kserve-ci-e2e-test resourceVersion: "14066" uid: a272b7ee-172e-4c08-a03b-655ec188d582 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-17T16:39:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-2a85d namespace: kserve-ci-e2e-test resourceVersion: "13344" uid: 57ea4952-aa89-483e-a366-5a6c21671ed4 kind: Event lastTimestamp: "2026-04-17T16:41:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:41:28Z" name: isvc-raw-sklearn-batcher-2a85d.18a732678931e644 namespace: kserve-ci-e2e-test resourceVersion: "14079" uid: ae5e575d-31e8-4f71-b62d-883d7a164420 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:39:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-2a85d namespace: kserve-ci-e2e-test resourceVersion: "13344" uid: 57ea4952-aa89-483e-a366-5a6c21671ed4 kind: Event lastTimestamp: "2026-04-17T16:39:58Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-2a85d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-2a85d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:39:58Z" name: isvc-raw-sklearn-batcher-2a85d.18a7326789d1976b namespace: kserve-ci-e2e-test resourceVersion: "13375" uid: cd054bd8-e42e-4210-93b3-d0db45ce1930 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:39:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-2a85d namespace: kserve-ci-e2e-test resourceVersion: "13344" uid: 57ea4952-aa89-483e-a366-5a6c21671ed4 kind: Event lastTimestamp: "2026-04-17T16:39:58Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-2a85d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:39:58Z" name: isvc-raw-sklearn-batcher-2a85d.18a7326789d1e883 namespace: kserve-ci-e2e-test resourceVersion: "13376" uid: 748c731f-1bc3-4db9-a913-aae0dcdc38ee reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-2a85d namespace: kserve-ci-e2e-test resourceVersion: "14076" uid: 57ea4952-aa89-483e-a366-5a6c21671ed4 kind: Event lastTimestamp: "2026-04-17T16:41:28Z" message: InferenceService [isvc-raw-sklearn-batcher-2a85d] is Ready metadata: creationTimestamp: "2026-04-17T16:41:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:41:28Z" name: isvc-raw-sklearn-batcher-2a85d.18a7327c9e863554 namespace: kserve-ci-e2e-test resourceVersion: "14077" uid: a40310cd-1cfd-4ebc-9b32-df4bc23300f9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:50:39.747578Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-c62bcc-predictor-6d9cfb45f7-5lf75 namespace: kserve-ci-e2e-test resourceVersion: "18927" uid: 68b64587-ef25-439b-b5bb-c1240151639b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-c62bcc-predictor-6d9cfb45f7-5lf75 to ip-10-0-135-127.ec2.internal metadata: creationTimestamp: "2026-04-17T16:50:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:50:39Z" name: isvc-secondary-c62bcc-predictor-6d9cfb45f7-5lf75.18a732fce34d3a15 namespace: kserve-ci-e2e-test resourceVersion: "18931" uid: 3120f9d9-774d-4a7f-9f05-0a70b9e958d9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f54646664-zzsrx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:50:40Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-c62bcc-predictor-6d9cfb45f7-5lf75 namespace: kserve-ci-e2e-test resourceVersion: "18935" uid: 68b64587-ef25-439b-b5bb-c1240151639b kind: Event lastTimestamp: "2026-04-17T16:50:40Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:50:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:50:40Z" name: isvc-secondary-c62bcc-predictor-6d9cfb45f7-5lf75.18a732fcfc3ec70a namespace: kserve-ci-e2e-test resourceVersion: "18943" uid: 7bb80129-fe0e-46ac-8038-e400c0e6de40 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:50:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-c62bcc-predictor-6d9cfb45f7-5lf75 namespace: kserve-ci-e2e-test resourceVersion: "18929" uid: 68b64587-ef25-439b-b5bb-c1240151639b kind: Event lastTimestamp: "2026-04-17T16:50:46Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0cd196d4c53b891914316f18ab5cfa9f85258e057f3687e65332c70bf642d22d" already present on machine metadata: creationTimestamp: "2026-04-17T16:50:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:50:46Z" name: isvc-secondary-c62bcc-predictor-6d9cfb45f7-5lf75.18a732fcfda1173b namespace: kserve-ci-e2e-test resourceVersion: "18995" uid: f7ae0062-58d2-4c46-8f0a-0f875f791ca9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:50:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-c62bcc-predictor-6d9cfb45f7-5lf75 namespace: kserve-ci-e2e-test resourceVersion: "18929" uid: 68b64587-ef25-439b-b5bb-c1240151639b kind: Event lastTimestamp: "2026-04-17T16:50:46Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-17T16:50:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:50:46Z" name: isvc-secondary-c62bcc-predictor-6d9cfb45f7-5lf75.18a732fd00b92ea5 namespace: kserve-ci-e2e-test resourceVersion: "18997" uid: 08878f39-330a-4de9-9f6d-0cf5ce046b13 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:50:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-c62bcc-predictor-6d9cfb45f7-5lf75 namespace: kserve-ci-e2e-test resourceVersion: "18929" uid: 68b64587-ef25-439b-b5bb-c1240151639b kind: Event lastTimestamp: "2026-04-17T16:50:46Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-17T16:50:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:50:46Z" name: isvc-secondary-c62bcc-predictor-6d9cfb45f7-5lf75.18a732fd012f2106 namespace: kserve-ci-e2e-test resourceVersion: "19000" uid: e2dc1d0f-5503-4c71-a628-06bf93d821fe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:50:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-c62bcc-predictor-6d9cfb45f7-5lf75 namespace: kserve-ci-e2e-test resourceVersion: "18929" uid: 68b64587-ef25-439b-b5bb-c1240151639b kind: Event lastTimestamp: "2026-04-17T16:50:50Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-c62bcc-predictor-6d9cfb45f7-5lf75_kserve-ci-e2e-test(68b64587-ef25-439b-b5bb-c1240151639b) metadata: creationTimestamp: "2026-04-17T16:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:50:50Z" name: isvc-secondary-c62bcc-predictor-6d9cfb45f7-5lf75.18a732ff70f8e049 namespace: kserve-ci-e2e-test resourceVersion: "19036" uid: 7036c99f-bc19-44f0-a9e0-5d60d911b0d7 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:50:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-c62bcc-predictor-6d9cfb45f7 namespace: kserve-ci-e2e-test resourceVersion: "18915" uid: 7a4d375c-db3f-414b-a697-de2f16f73da2 kind: Event lastTimestamp: "2026-04-17T16:50:39Z" message: 'Created pod: isvc-secondary-c62bcc-predictor-6d9cfb45f7-5lf75' metadata: creationTimestamp: "2026-04-17T16:50:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:50:39Z" name: isvc-secondary-c62bcc-predictor-6d9cfb45f7.18a732fce2875e21 namespace: kserve-ci-e2e-test resourceVersion: "18928" uid: 3c6aef56-5596-4459-89f1-f4909f5d0ecf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:50:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-c62bcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18913" uid: c49f3a2c-928c-4468-ab66-0d679daa8fe9 kind: Event lastTimestamp: "2026-04-17T16:50:39Z" message: Scaled up replica set isvc-secondary-c62bcc-predictor-6d9cfb45f7 from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:50:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:50:39Z" name: isvc-secondary-c62bcc-predictor.18a732fcdeeca184 namespace: kserve-ci-e2e-test resourceVersion: "18917" uid: 54b4cdcd-bb98-4294-831b-bcd0391e7b2b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:50:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-c62bcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18914" uid: a15a0aba-ffb0-4d64-bd18-b1ad14e041c9 kind: Event lastTimestamp: "2026-04-17T16:50:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:50:54Z" name: isvc-secondary-c62bcc-predictor.18a733005d627fb6 namespace: kserve-ci-e2e-test resourceVersion: "19070" uid: 96ba738e-06a4-49d4-884d-051efe9e2b50 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:50:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-c62bcc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18914" uid: a15a0aba-ffb0-4d64-bd18-b1ad14e041c9 kind: Event lastTimestamp: "2026-04-17T16:50:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:50:54Z" name: isvc-secondary-c62bcc-predictor.18a733005dc97efe namespace: kserve-ci-e2e-test resourceVersion: "19072" uid: e1cd6c9b-bfc6-44ae-8627-d1820dbdc1bd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-17T16:50:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-c62bcc namespace: kserve-ci-e2e-test resourceVersion: "18903" uid: 987da21e-25d7-41a8-8588-f2ffaed13320 kind: Event lastTimestamp: "2026-04-17T16:50:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:50:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:50:53Z" name: isvc-secondary-c62bcc.18a732fcdec6347b namespace: kserve-ci-e2e-test resourceVersion: "19058" uid: 778fbc7b-4da8-4c46-822b-85fde5a618a6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:50:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-c62bcc namespace: kserve-ci-e2e-test resourceVersion: "18903" uid: 987da21e-25d7-41a8-8588-f2ffaed13320 kind: Event lastTimestamp: "2026-04-17T16:50:39Z" message: 'Failed to update status for InferenceService "isvc-secondary-c62bcc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-c62bcc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:50:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:50:39Z" name: isvc-secondary-c62bcc.18a732fcdf1a36fc namespace: kserve-ci-e2e-test resourceVersion: "18937" uid: 35da4b57-27be-4d55-bdef-60dbc6bedd5f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:50:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-c62bcc namespace: kserve-ci-e2e-test resourceVersion: "18903" uid: 987da21e-25d7-41a8-8588-f2ffaed13320 kind: Event lastTimestamp: "2026-04-17T16:50:39Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-c62bcc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:50:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:50:39Z" name: isvc-secondary-c62bcc.18a732fcdf1a7ea8 namespace: kserve-ci-e2e-test resourceVersion: "18938" uid: b73350a1-73b7-481a-9a68-53bcf5290e1d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:41:43.469240Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k namespace: kserve-ci-e2e-test resourceVersion: "14224" uid: 7c3b027b-aaf4-4013-b294-9634cef66a61 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k to ip-10-0-135-127.ec2.internal metadata: creationTimestamp: "2026-04-17T16:41:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:41:43Z" name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k.18a73280069f34eb namespace: kserve-ci-e2e-test resourceVersion: "14230" uid: 4e51a858-749c-44ec-a5c2-06111eb837b0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f54646664-zzsrx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:43Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k namespace: kserve-ci-e2e-test resourceVersion: "14241" uid: 7c3b027b-aaf4-4013-b294-9634cef66a61 kind: Event lastTimestamp: "2026-04-17T16:41:43Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:41:43Z" name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k.18a732801f67b3f1 namespace: kserve-ci-e2e-test resourceVersion: "14270" uid: afa73623-d5d7-4c58-a58f-fd864adc45f4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k namespace: kserve-ci-e2e-test resourceVersion: "14226" uid: 7c3b027b-aaf4-4013-b294-9634cef66a61 kind: Event lastTimestamp: "2026-04-17T16:41:43Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0cd196d4c53b891914316f18ab5cfa9f85258e057f3687e65332c70bf642d22d" already present on machine metadata: creationTimestamp: "2026-04-17T16:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:43Z" name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k.18a7328020ca5d12 namespace: kserve-ci-e2e-test resourceVersion: "14272" uid: 8f4e91fa-43b7-4e8d-bd0b-8d6da010abf4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k namespace: kserve-ci-e2e-test resourceVersion: "14226" uid: 7c3b027b-aaf4-4013-b294-9634cef66a61 kind: Event lastTimestamp: "2026-04-17T16:41:43Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-17T16:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:43Z" name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k.18a7328024562674 namespace: kserve-ci-e2e-test resourceVersion: "14274" uid: c2a561bc-9cac-4843-aa8b-df75342ac126 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k namespace: kserve-ci-e2e-test resourceVersion: "14226" uid: 7c3b027b-aaf4-4013-b294-9634cef66a61 kind: Event lastTimestamp: "2026-04-17T16:41:43Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-17T16:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:43Z" name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k.18a73280251c5560 namespace: kserve-ci-e2e-test resourceVersion: "14275" uid: 152764e8-f6e9-48af-a8c6-204253acdbd6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k namespace: kserve-ci-e2e-test resourceVersion: "14226" uid: 7c3b027b-aaf4-4013-b294-9634cef66a61 kind: Event lastTimestamp: "2026-04-17T16:41:47Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T16:41:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:47Z" name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k.18a73281136387bf namespace: kserve-ci-e2e-test resourceVersion: "14326" uid: 0c29ea5c-1650-4a55-856c-60cbfa74ceaf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k namespace: kserve-ci-e2e-test resourceVersion: "14226" uid: 7c3b027b-aaf4-4013-b294-9634cef66a61 kind: Event lastTimestamp: "2026-04-17T16:41:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:48Z" name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k.18a732811737e9e0 namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: ecb2f12a-3b1c-4224-a429-71c19aab1551 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k namespace: kserve-ci-e2e-test resourceVersion: "14226" uid: 7c3b027b-aaf4-4013-b294-9634cef66a61 kind: Event lastTimestamp: "2026-04-17T16:41:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:48Z" name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k.18a73281179fd3eb namespace: kserve-ci-e2e-test resourceVersion: "14334" uid: 5b06722a-be9c-44a4-8160-92874723f56f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k namespace: kserve-ci-e2e-test resourceVersion: "14226" uid: 7c3b027b-aaf4-4013-b294-9634cef66a61 kind: Event lastTimestamp: "2026-04-17T16:41:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T16:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:48Z" name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k.18a7328117acac98 namespace: kserve-ci-e2e-test resourceVersion: "14335" uid: d18bcd29-71b0-4df4-85d8-66f457dc01e5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k namespace: kserve-ci-e2e-test resourceVersion: "14226" uid: 7c3b027b-aaf4-4013-b294-9634cef66a61 kind: Event lastTimestamp: "2026-04-17T16:41:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:48Z" name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k.18a732811b240919 namespace: kserve-ci-e2e-test resourceVersion: "14338" uid: 36bc576f-07ac-46ae-b87a-d9d776f645c7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k namespace: kserve-ci-e2e-test resourceVersion: "14226" uid: 7c3b027b-aaf4-4013-b294-9634cef66a61 kind: Event lastTimestamp: "2026-04-17T16:41:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:48Z" name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k.18a732811b874dc3 namespace: kserve-ci-e2e-test resourceVersion: "14340" uid: 17959f9b-fe4c-49bb-9416-803d03e51759 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-17T16:41:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k namespace: kserve-ci-e2e-test resourceVersion: "14226" uid: 7c3b027b-aaf4-4013-b294-9634cef66a61 kind: Event lastTimestamp: "2026-04-17T16:43:34Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:35Z" name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k.18a732814f954513 namespace: kserve-ci-e2e-test resourceVersion: "15348" uid: dcf0a447-3260-43b7-8ec6-266f4e6bdd8a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k namespace: kserve-ci-e2e-test resourceVersion: "14226" uid: 7c3b027b-aaf4-4013-b294-9634cef66a61 kind: Event lastTimestamp: "2026-04-17T16:43:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T16:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:33Z" name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k.18a73299af628ada namespace: kserve-ci-e2e-test resourceVersion: "15254" uid: 26856345-e6b2-44e2-a24d-a3adb44df697 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k namespace: kserve-ci-e2e-test resourceVersion: "14226" uid: 7c3b027b-aaf4-4013-b294-9634cef66a61 kind: Event lastTimestamp: "2026-04-17T16:43:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:33Z" name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k.18a73299af654554 namespace: kserve-ci-e2e-test resourceVersion: "15257" uid: 8b3cec1a-a7e1-43f5-9dde-ce1e6766e3b9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k namespace: kserve-ci-e2e-test resourceVersion: "14226" uid: 7c3b027b-aaf4-4013-b294-9634cef66a61 kind: Event lastTimestamp: "2026-04-17T16:43:34Z" message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": dial tcp 10.134.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:34Z" name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k.18a73299fde2b2a2 namespace: kserve-ci-e2e-test resourceVersion: "15347" uid: 447a90f0-a174-4cbb-abe8-4f429b248903 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7 namespace: kserve-ci-e2e-test resourceVersion: "14206" uid: 02176544-af85-4ce7-ae34-eaaf4bbf00bc kind: Event lastTimestamp: "2026-04-17T16:41:43Z" message: 'Created pod: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7-jpg8k' metadata: creationTimestamp: "2026-04-17T16:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:41:43Z" name: isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7.18a73280060b9ed1 namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: bfff0ddb-e01a-4c89-945a-4c167db5e0cf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-f88b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "14203" uid: e311186d-d0a3-4b74-9fef-e1be5edf0f73 kind: Event lastTimestamp: "2026-04-17T16:41:43Z" message: Scaled up replica set isvc-sklearn-graph-raw-f88b3-predictor-65f4fbb7f7 from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:41:43Z" name: isvc-sklearn-graph-raw-f88b3-predictor.18a7327ffe504a9b namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 295ef61b-1ab1-4113-b7af-51d7f43460f0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:41:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-f88b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "14205" uid: d9a9723c-3d53-4a02-a49b-7cead294a5ed kind: Event lastTimestamp: "2026-04-17T16:42:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:41:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:42:13Z" name: isvc-sklearn-graph-raw-f88b3-predictor.18a732837cc58ac9 namespace: kserve-ci-e2e-test resourceVersion: "14544" uid: 68603eff-da6b-4687-b656-97e1a29f8fbf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:41:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-f88b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "14205" uid: d9a9723c-3d53-4a02-a49b-7cead294a5ed kind: Event lastTimestamp: "2026-04-17T16:42:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:41:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:42:13Z" name: isvc-sklearn-graph-raw-f88b3-predictor.18a732837d29ed35 namespace: kserve-ci-e2e-test resourceVersion: "14545" uid: 43f0bcd6-1f43-4086-a513-79a3586f0d54 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T16:42:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-f88b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "14419" uid: d9a9723c-3d53-4a02-a49b-7cead294a5ed kind: Event lastTimestamp: "2026-04-17T16:43:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:42:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:28Z" name: isvc-sklearn-graph-raw-f88b3-predictor.18a7328a7b0d12b1 namespace: kserve-ci-e2e-test resourceVersion: "15151" uid: 2f9ce5bd-55b7-4259-9bb7-7cad118cc9e4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T16:42:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-f88b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "14419" uid: d9a9723c-3d53-4a02-a49b-7cead294a5ed kind: Event lastTimestamp: "2026-04-17T16:43:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:42:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:28Z" name: isvc-sklearn-graph-raw-f88b3-predictor.18a7328a7b73de56 namespace: kserve-ci-e2e-test resourceVersion: "15152" uid: 60527640-d69e-4877-b97c-a2c126c74b23 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-17T16:41:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-f88b3 namespace: kserve-ci-e2e-test resourceVersion: "14179" uid: 4d1a9335-1ea8-4817-9384-88b4e9b4b33f kind: Event lastTimestamp: "2026-04-17T16:43:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:43:05Z" name: isvc-sklearn-graph-raw-f88b3.18a7327ffe0899ea namespace: kserve-ci-e2e-test resourceVersion: "14943" uid: d33b2f70-cc20-4fdf-9c62-52368ae8d806 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-f88b3 namespace: kserve-ci-e2e-test resourceVersion: "14179" uid: 4d1a9335-1ea8-4817-9384-88b4e9b4b33f kind: Event lastTimestamp: "2026-04-17T16:41:43Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-f88b3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-f88b3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:41:43Z" name: isvc-sklearn-graph-raw-f88b3.18a7327ffe89003f namespace: kserve-ci-e2e-test resourceVersion: "14211" uid: 4ff70a9f-b9f0-4b3d-bb10-109a23fe24e1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-f88b3 namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: 4d1a9335-1ea8-4817-9384-88b4e9b4b33f kind: Event lastTimestamp: "2026-04-17T16:43:05Z" message: InferenceService [isvc-sklearn-graph-raw-f88b3] is Ready metadata: creationTimestamp: "2026-04-17T16:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:43:05Z" name: isvc-sklearn-graph-raw-f88b3.18a7329306cd8d08 namespace: kserve-ci-e2e-test resourceVersion: "14942" uid: 013dfa1c-ec31-4014-a60e-07d4956f4013 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:43:33.766284Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f namespace: kserve-ci-e2e-test resourceVersion: "15264" uid: e7f33a07-6e34-4ae0-be9e-ee66e6a59b01 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f to ip-10-0-135-127.ec2.internal metadata: creationTimestamp: "2026-04-17T16:43:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:43:33Z" name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f.18a73299b4d679b7 namespace: kserve-ci-e2e-test resourceVersion: "15275" uid: 3fe8a094-1c8f-48b4-b5ac-f5f47d81ea6a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f54646664-zzsrx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:34Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: e7f33a07-6e34-4ae0-be9e-ee66e6a59b01 kind: Event lastTimestamp: "2026-04-17T16:43:34Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:43:34Z" name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f.18a73299cd94b466 namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: c73abe28-3b4b-43ec-b5a9-6c1a5aa9d5e6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: e7f33a07-6e34-4ae0-be9e-ee66e6a59b01 kind: Event lastTimestamp: "2026-04-17T16:43:34Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0cd196d4c53b891914316f18ab5cfa9f85258e057f3687e65332c70bf642d22d" already present on machine metadata: creationTimestamp: "2026-04-17T16:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:34Z" name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f.18a73299cf0538c7 namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 431af9ae-7e36-4120-ae9f-e59dd7ce214d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: e7f33a07-6e34-4ae0-be9e-ee66e6a59b01 kind: Event lastTimestamp: "2026-04-17T16:43:34Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-17T16:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:34Z" name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f.18a73299d1e3165d namespace: kserve-ci-e2e-test resourceVersion: "15328" uid: 5b68ef80-4d3d-4692-90bf-89a2d03ef71b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: e7f33a07-6e34-4ae0-be9e-ee66e6a59b01 kind: Event lastTimestamp: "2026-04-17T16:43:34Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-17T16:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:34Z" name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f.18a73299d25aebb1 namespace: kserve-ci-e2e-test resourceVersion: "15329" uid: 1a553aee-0bbc-4c32-b2ee-525f4ac975be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: e7f33a07-6e34-4ae0-be9e-ee66e6a59b01 kind: Event lastTimestamp: "2026-04-17T16:43:38Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T16:43:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:38Z" name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f.18a7329ac569c4fc namespace: kserve-ci-e2e-test resourceVersion: "15386" uid: b93b24a9-9b61-41f2-a3f2-e774cb45fc99 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: e7f33a07-6e34-4ae0-be9e-ee66e6a59b01 kind: Event lastTimestamp: "2026-04-17T16:43:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:43:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:38Z" name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f.18a7329ac932300e namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: c6151bfb-239a-47c8-80cf-2b41c4bf9f7e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: e7f33a07-6e34-4ae0-be9e-ee66e6a59b01 kind: Event lastTimestamp: "2026-04-17T16:43:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:43:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:38Z" name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f.18a7329ac9b91641 namespace: kserve-ci-e2e-test resourceVersion: "15393" uid: 557de5e4-7d5a-4ea4-ad79-39a77ad5a969 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: e7f33a07-6e34-4ae0-be9e-ee66e6a59b01 kind: Event lastTimestamp: "2026-04-17T16:43:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T16:43:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:38Z" name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f.18a7329ac9c5a890 namespace: kserve-ci-e2e-test resourceVersion: "15394" uid: 31fa9c51-9d55-480b-95d6-c416712902df reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: e7f33a07-6e34-4ae0-be9e-ee66e6a59b01 kind: Event lastTimestamp: "2026-04-17T16:43:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:43:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:38Z" name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f.18a7329acc45727a namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 802584e7-6592-4b1f-b0be-ac298c202e7f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: e7f33a07-6e34-4ae0-be9e-ee66e6a59b01 kind: Event lastTimestamp: "2026-04-17T16:43:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:43:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:38Z" name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f.18a7329accb6883d namespace: kserve-ci-e2e-test resourceVersion: "15398" uid: 284d8bb9-549b-4318-a476-50868ace0603 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-17T16:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: e7f33a07-6e34-4ae0-be9e-ee66e6a59b01 kind: Event lastTimestamp: "2026-04-17T16:45:15Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:15Z" name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f.18a7329b01adee7a namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: f23a577f-9b89-441a-ac93-f3218c73713a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: e7f33a07-6e34-4ae0-be9e-ee66e6a59b01 kind: Event lastTimestamp: "2026-04-17T16:45:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T16:45:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:13Z" name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f.18a732b1092a56dc namespace: kserve-ci-e2e-test resourceVersion: "16225" uid: b02fb8dd-f092-41b8-8830-6673629dc98e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: e7f33a07-6e34-4ae0-be9e-ee66e6a59b01 kind: Event lastTimestamp: "2026-04-17T16:45:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:45:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:13Z" name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f.18a732b1092ad1ff namespace: kserve-ci-e2e-test resourceVersion: "16226" uid: c862da67-f94f-440c-9cf2-85595d333f62 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: e7f33a07-6e34-4ae0-be9e-ee66e6a59b01 kind: Event lastTimestamp: "2026-04-17T16:45:15Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:15Z" name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f.18a732b15c471464 namespace: kserve-ci-e2e-test resourceVersion: "16276" uid: 5a7f876c-b2e2-4272-8e58-6e753b52e2ec reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985 namespace: kserve-ci-e2e-test resourceVersion: "15243" uid: a890a23e-02c4-4b1d-9ae6-bee895ed5d1b kind: Event lastTimestamp: "2026-04-17T16:43:33Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985-zjk5f' metadata: creationTimestamp: "2026-04-17T16:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:33Z" name: isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985.18a73299b409edca namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: 414365ab-9a22-4d40-9182-7165a52ac908 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-10815-predictor namespace: kserve-ci-e2e-test resourceVersion: "15236" uid: 24b86013-f22a-4ca1-9cbf-e5b1955a9f4d kind: Event lastTimestamp: "2026-04-17T16:43:33Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-10815-predictor-57dfb49985 from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:33Z" name: isvc-sklearn-graph-raw-hpa-10815-predictor.18a73299ae7a5c5a namespace: kserve-ci-e2e-test resourceVersion: "15245" uid: cbe74153-a625-4441-aff7-ded028f199fd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:43:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-10815-predictor namespace: kserve-ci-e2e-test resourceVersion: "15241" uid: c43359aa-3a17-4276-927a-db6eb368e865 kind: Event lastTimestamp: "2026-04-17T16:44:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:43:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:44:03Z" name: isvc-sklearn-graph-raw-hpa-10815-predictor.18a7329d2d5ac585 namespace: kserve-ci-e2e-test resourceVersion: "15608" uid: 79502516-fad3-49d1-abfc-fef656f6b5bc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:43:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-10815-predictor namespace: kserve-ci-e2e-test resourceVersion: "15241" uid: c43359aa-3a17-4276-927a-db6eb368e865 kind: Event lastTimestamp: "2026-04-17T16:44:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:43:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:44:03Z" name: isvc-sklearn-graph-raw-hpa-10815-predictor.18a7329d2dbc9a80 namespace: kserve-ci-e2e-test resourceVersion: "15609" uid: 41f6d43c-1652-4cf9-a017-20ea7ca9816a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T16:44:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-10815-predictor namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: c43359aa-3a17-4276-927a-db6eb368e865 kind: Event lastTimestamp: "2026-04-17T16:45:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:44:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:45:03Z" name: isvc-sklearn-graph-raw-hpa-10815-predictor.18a732a42bce7bbc namespace: kserve-ci-e2e-test resourceVersion: "16071" uid: c1f7fff7-47bf-4ead-b816-eee3591a1f18 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T16:44:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-10815-predictor namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: c43359aa-3a17-4276-927a-db6eb368e865 kind: Event lastTimestamp: "2026-04-17T16:45:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:44:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:45:03Z" name: isvc-sklearn-graph-raw-hpa-10815-predictor.18a732a42c34b0aa namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: 122aa1fd-d7f3-44d1-8877-067caea18b40 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-17T16:43:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-10815 namespace: kserve-ci-e2e-test resourceVersion: "15218" uid: 704738c9-791a-4f17-85c3-7c4ddde5f943 kind: Event lastTimestamp: "2026-04-17T16:44:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:44:45Z" name: isvc-sklearn-graph-raw-hpa-10815.18a73299ae51238f namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: 33d06e5d-751e-4e11-9304-d5c2d2979dd5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-10815 namespace: kserve-ci-e2e-test resourceVersion: "15218" uid: 704738c9-791a-4f17-85c3-7c4ddde5f943 kind: Event lastTimestamp: "2026-04-17T16:43:33Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-10815": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-10815": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:43:33Z" name: isvc-sklearn-graph-raw-hpa-10815.18a73299aea94a84 namespace: kserve-ci-e2e-test resourceVersion: "15249" uid: b738094f-eac1-4aa8-832e-33cd144e7b2d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:44:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-10815 namespace: kserve-ci-e2e-test resourceVersion: "15919" uid: 704738c9-791a-4f17-85c3-7c4ddde5f943 kind: Event lastTimestamp: "2026-04-17T16:44:45Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-10815] is Ready metadata: creationTimestamp: "2026-04-17T16:44:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:44:45Z" name: isvc-sklearn-graph-raw-hpa-10815.18a732aa657a0172 namespace: kserve-ci-e2e-test resourceVersion: "15920" uid: dd5a233a-aac0-4d9f-81b5-dce67a2828ba reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:46:49.304518Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz namespace: kserve-ci-e2e-test resourceVersion: "17099" uid: 99cd5cac-5bbd-4f52-ac82-88b47eeb7183 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz to ip-10-0-135-127.ec2.internal metadata: creationTimestamp: "2026-04-17T16:46:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:46:49Z" name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz.18a732c73bd333f9 namespace: kserve-ci-e2e-test resourceVersion: "17104" uid: 28db76d2-6a16-47dc-b55a-02c882b757af reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f54646664-zzsrx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:46:49Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 99cd5cac-5bbd-4f52-ac82-88b47eeb7183 kind: Event lastTimestamp: "2026-04-17T16:46:49Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-scale-raw-b7bd1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T16:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:46:49Z" name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz.18a732c74868f812 namespace: kserve-ci-e2e-test resourceVersion: "17130" uid: e508fc77-249f-43d5-bf41-f6d619977e98 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:46:50Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz namespace: kserve-ci-e2e-test resourceVersion: "17113" uid: 99cd5cac-5bbd-4f52-ac82-88b47eeb7183 kind: Event lastTimestamp: "2026-04-17T16:46:50Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:46:50Z" name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz.18a732c778b051ea namespace: kserve-ci-e2e-test resourceVersion: "17144" uid: 1477619b-fa68-4049-bee2-6c836d0b8d8a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:46:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 99cd5cac-5bbd-4f52-ac82-88b47eeb7183 kind: Event lastTimestamp: "2026-04-17T16:46:50Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0cd196d4c53b891914316f18ab5cfa9f85258e057f3687e65332c70bf642d22d" already present on machine metadata: creationTimestamp: "2026-04-17T16:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:46:50Z" name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz.18a732c77a10aa25 namespace: kserve-ci-e2e-test resourceVersion: "17146" uid: d0bc71e2-2b0f-428f-8d21-e14e67e2f281 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:46:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 99cd5cac-5bbd-4f52-ac82-88b47eeb7183 kind: Event lastTimestamp: "2026-04-17T16:46:50Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-17T16:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:46:50Z" name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz.18a732c77cf663e1 namespace: kserve-ci-e2e-test resourceVersion: "17147" uid: 127655de-b45c-46bd-9183-849ba2b2f178 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:46:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 99cd5cac-5bbd-4f52-ac82-88b47eeb7183 kind: Event lastTimestamp: "2026-04-17T16:46:50Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-17T16:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:46:50Z" name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz.18a732c77d64bb95 namespace: kserve-ci-e2e-test resourceVersion: "17148" uid: 75764974-9312-4f52-bb0e-6d1d24bb142e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:46:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 99cd5cac-5bbd-4f52-ac82-88b47eeb7183 kind: Event lastTimestamp: "2026-04-17T16:46:53Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T16:46:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:46:53Z" name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz.18a732c8521931a5 namespace: kserve-ci-e2e-test resourceVersion: "17181" uid: 29b53d27-b569-4085-a418-7205b6a5021b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:46:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 99cd5cac-5bbd-4f52-ac82-88b47eeb7183 kind: Event lastTimestamp: "2026-04-17T16:46:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:46:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:46:54Z" name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz.18a732c855337ef1 namespace: kserve-ci-e2e-test resourceVersion: "17185" uid: 492e2342-27d2-43ac-bb1f-fa39d9384ddb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:46:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 99cd5cac-5bbd-4f52-ac82-88b47eeb7183 kind: Event lastTimestamp: "2026-04-17T16:46:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:46:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:46:54Z" name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz.18a732c855ff22d2 namespace: kserve-ci-e2e-test resourceVersion: "17186" uid: 51344c08-f970-45b5-b8b1-a1ddf8d9c606 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:46:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 99cd5cac-5bbd-4f52-ac82-88b47eeb7183 kind: Event lastTimestamp: "2026-04-17T16:46:54Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T16:46:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:46:54Z" name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz.18a732c8560bd296 namespace: kserve-ci-e2e-test resourceVersion: "17188" uid: c4ff7e52-79ce-4479-a829-fe79e9baef2e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:46:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 99cd5cac-5bbd-4f52-ac82-88b47eeb7183 kind: Event lastTimestamp: "2026-04-17T16:46:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:46:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:46:54Z" name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz.18a732c858f65b55 namespace: kserve-ci-e2e-test resourceVersion: "17190" uid: 8ff62e80-851b-40a3-a743-2392bc726d86 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:46:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 99cd5cac-5bbd-4f52-ac82-88b47eeb7183 kind: Event lastTimestamp: "2026-04-17T16:46:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:46:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:46:54Z" name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz.18a732c8595d28b0 namespace: kserve-ci-e2e-test resourceVersion: "17191" uid: a5e611e2-0020-48a7-8bdb-e7e0c7cdb118 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 16 eventTime: null firstTimestamp: "2026-04-17T16:46:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 99cd5cac-5bbd-4f52-ac82-88b47eeb7183 kind: Event lastTimestamp: "2026-04-17T16:49:09Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:49:09Z" name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz.18a732c8ca39ecc7 namespace: kserve-ci-e2e-test resourceVersion: "18158" uid: 6f7ae3e2-947f-471f-b465-7d197691886e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:49:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 99cd5cac-5bbd-4f52-ac82-88b47eeb7183 kind: Event lastTimestamp: "2026-04-17T16:49:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T16:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:49:19Z" name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz.18a732ea336c5805 namespace: kserve-ci-e2e-test resourceVersion: "18267" uid: 67cecc0f-5a54-4431-85c4-78612036479d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:49:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 99cd5cac-5bbd-4f52-ac82-88b47eeb7183 kind: Event lastTimestamp: "2026-04-17T16:49:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:49:19Z" name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz.18a732ea336cfea1 namespace: kserve-ci-e2e-test resourceVersion: "18270" uid: c1dfd57d-6b4e-46fd-8ba0-df4a9d3a4411 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:49:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 99cd5cac-5bbd-4f52-ac82-88b47eeb7183 kind: Event lastTimestamp: "2026-04-17T16:49:26Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:49:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:49:26Z" name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz.18a732eac8b1c26e namespace: kserve-ci-e2e-test resourceVersion: "18366" uid: 8f1442a1-9ca3-4949-97c9-af78e2f58e12 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:46:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5 namespace: kserve-ci-e2e-test resourceVersion: "17076" uid: 99fb6a95-8a34-43bf-8bec-ab56e0c92be5 kind: Event lastTimestamp: "2026-04-17T16:46:49Z" message: 'Created pod: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5-ps2dz' metadata: creationTimestamp: "2026-04-17T16:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:46:49Z" name: isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5.18a732c73b5e3d92 namespace: kserve-ci-e2e-test resourceVersion: "17102" uid: f6347ed7-89f0-4921-9d23-a853ca0a326d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:46:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-b7bd1-predictor namespace: kserve-ci-e2e-test resourceVersion: "17071" uid: 46cfda44-bcfa-47ce-bdd7-c27dca1e399a kind: Event lastTimestamp: "2026-04-17T16:46:49Z" message: Scaled up replica set isvc-sklearn-scale-raw-b7bd1-predictor-68447964c5 from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:46:49Z" name: isvc-sklearn-scale-raw-b7bd1-predictor.18a732c7375ffd62 namespace: kserve-ci-e2e-test resourceVersion: "17082" uid: e75aafcf-11c7-43e7-9dde-5ee8c62f806e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:47:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-b7bd1-predictor namespace: kserve-ci-e2e-test resourceVersion: "17075" uid: 78e76ed6-f0b1-4c84-98e8-229b3ebfaac8 kind: Event lastTimestamp: "2026-04-17T16:47:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:47:34Z" name: isvc-sklearn-scale-raw-b7bd1-predictor.18a732cab5da5523 namespace: kserve-ci-e2e-test resourceVersion: "17501" uid: eb4b2815-d9d5-43f6-8bd4-89358cf0dd67 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:47:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-b7bd1-predictor namespace: kserve-ci-e2e-test resourceVersion: "17075" uid: 78e76ed6-f0b1-4c84-98e8-229b3ebfaac8 kind: Event lastTimestamp: "2026-04-17T16:47:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:47:34Z" name: isvc-sklearn-scale-raw-b7bd1-predictor.18a732cab664d783 namespace: kserve-ci-e2e-test resourceVersion: "17502" uid: b09ab14a-cc22-45ad-a6d5-7b525777943e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T16:47:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-b7bd1-predictor namespace: kserve-ci-e2e-test resourceVersion: "17275" uid: 78e76ed6-f0b1-4c84-98e8-229b3ebfaac8 kind: Event lastTimestamp: "2026-04-17T16:49:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:49:19Z" name: isvc-sklearn-scale-raw-b7bd1-predictor.18a732d5339bf4d0 namespace: kserve-ci-e2e-test resourceVersion: "18238" uid: b19d6785-787b-467d-bcd3-ff1f0745a36e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T16:47:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-b7bd1-predictor namespace: kserve-ci-e2e-test resourceVersion: "17275" uid: 78e76ed6-f0b1-4c84-98e8-229b3ebfaac8 kind: Event lastTimestamp: "2026-04-17T16:49:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:49:19Z" name: isvc-sklearn-scale-raw-b7bd1-predictor.18a732d533ffd10c namespace: kserve-ci-e2e-test resourceVersion: "18239" uid: b7ea40d7-9910-4612-9d03-4895520e472d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-17T16:46:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-b7bd1 namespace: kserve-ci-e2e-test resourceVersion: "17058" uid: d3fda005-18d6-4445-ad3e-1b340f2ec331 kind: Event lastTimestamp: "2026-04-17T16:49:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:49:19Z" name: isvc-sklearn-scale-raw-b7bd1.18a732c7373a96bc namespace: kserve-ci-e2e-test resourceVersion: "18235" uid: 3f9b7367-3d31-4ee8-a288-356239c9a3f7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:46:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-b7bd1 namespace: kserve-ci-e2e-test resourceVersion: "17058" uid: d3fda005-18d6-4445-ad3e-1b340f2ec331 kind: Event lastTimestamp: "2026-04-17T16:46:49Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-b7bd1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-b7bd1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:46:49Z" name: isvc-sklearn-scale-raw-b7bd1.18a732c737d3a06a namespace: kserve-ci-e2e-test resourceVersion: "17122" uid: 7565d07b-c9e7-4347-ac37-a573f6a341d0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:46:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-b7bd1 namespace: kserve-ci-e2e-test resourceVersion: "17073" uid: d3fda005-18d6-4445-ad3e-1b340f2ec331 kind: Event lastTimestamp: "2026-04-17T16:46:49Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-b7bd1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:46:49Z" name: isvc-sklearn-scale-raw-b7bd1.18a732c73dc83186 namespace: kserve-ci-e2e-test resourceVersion: "17128" uid: 86e6e2c1-d63e-4f09-80e0-8b5c521da5f5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:49:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-b7bd1 namespace: kserve-ci-e2e-test resourceVersion: "18233" uid: d3fda005-18d6-4445-ad3e-1b340f2ec331 kind: Event lastTimestamp: "2026-04-17T16:49:19Z" message: InferenceService [isvc-sklearn-scale-raw-b7bd1] is Ready metadata: creationTimestamp: "2026-04-17T16:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:49:19Z" name: isvc-sklearn-scale-raw-b7bd1.18a732ea1dd5de3c namespace: kserve-ci-e2e-test resourceVersion: "18234" uid: 56d7f955-e3bf-418e-b4ec-885c7c288fd0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:41:43.561832Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j namespace: kserve-ci-e2e-test resourceVersion: "14254" uid: 472fed14-7cb5-47c1-8f39-8fbe93f72718 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j to ip-10-0-135-127.ec2.internal metadata: creationTimestamp: "2026-04-17T16:41:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:41:43Z" name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j.18a732800c241a1f namespace: kserve-ci-e2e-test resourceVersion: "14259" uid: 33683a26-2891-4377-9312-21efc1a1ee11 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f54646664-zzsrx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:43Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j namespace: kserve-ci-e2e-test resourceVersion: "14264" uid: 472fed14-7cb5-47c1-8f39-8fbe93f72718 kind: Event lastTimestamp: "2026-04-17T16:41:43Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:41:43Z" name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j.18a7328025d996c4 namespace: kserve-ci-e2e-test resourceVersion: "14276" uid: 7a7e3d79-4dd4-47ec-86c1-948a12526879 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 472fed14-7cb5-47c1-8f39-8fbe93f72718 kind: Event lastTimestamp: "2026-04-17T16:41:44Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0cd196d4c53b891914316f18ab5cfa9f85258e057f3687e65332c70bf642d22d" already present on machine metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:44Z" name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j.18a73280273cd4ef namespace: kserve-ci-e2e-test resourceVersion: "14278" uid: b0be4f64-e441-407e-9d3a-b96fd4d28918 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 472fed14-7cb5-47c1-8f39-8fbe93f72718 kind: Event lastTimestamp: "2026-04-17T16:41:44Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:44Z" name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j.18a732802a154a83 namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 92ea2d43-9d6a-4ee8-b403-2717c8816b0a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 472fed14-7cb5-47c1-8f39-8fbe93f72718 kind: Event lastTimestamp: "2026-04-17T16:41:44Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-17T16:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:44Z" name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j.18a732802a7e285b namespace: kserve-ci-e2e-test resourceVersion: "14280" uid: b09758d6-2954-46d5-a54f-fb16a5f9e104 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 472fed14-7cb5-47c1-8f39-8fbe93f72718 kind: Event lastTimestamp: "2026-04-17T16:41:47Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-17T16:41:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:41:47Z" name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j.18a73281137aaaed namespace: kserve-ci-e2e-test resourceVersion: "14328" uid: c1b726ec-987a-4c40-a4ae-b1291cd53b3e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 472fed14-7cb5-47c1-8f39-8fbe93f72718 kind: Event lastTimestamp: "2026-04-17T16:42:07Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 19.431s (19.431s including waiting). Image size: 1306229499 bytes.' metadata: creationTimestamp: "2026-04-17T16:42:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:07Z" name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j.18a7328599a977cd namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: 313d0bee-a147-4838-8fa5-446c97708640 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 472fed14-7cb5-47c1-8f39-8fbe93f72718 kind: Event lastTimestamp: "2026-04-17T16:42:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:42:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:07Z" name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j.18a732859d189e0a namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: e083ee98-d9a4-4f60-9232-413d06346565 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 472fed14-7cb5-47c1-8f39-8fbe93f72718 kind: Event lastTimestamp: "2026-04-17T16:42:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:42:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:07Z" name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j.18a732859d8a0544 namespace: kserve-ci-e2e-test resourceVersion: "14490" uid: 75c485df-cb20-4d62-81eb-b23aed000ff6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 472fed14-7cb5-47c1-8f39-8fbe93f72718 kind: Event lastTimestamp: "2026-04-17T16:42:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T16:42:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:07Z" name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j.18a732859d960221 namespace: kserve-ci-e2e-test resourceVersion: "14491" uid: 95e7373b-5e15-4758-bd3d-c1efe573e406 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 472fed14-7cb5-47c1-8f39-8fbe93f72718 kind: Event lastTimestamp: "2026-04-17T16:42:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:42:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:07Z" name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j.18a73285a069ed9e namespace: kserve-ci-e2e-test resourceVersion: "14492" uid: 0f621298-7c79-46bc-9b90-69e00335214d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:42:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 472fed14-7cb5-47c1-8f39-8fbe93f72718 kind: Event lastTimestamp: "2026-04-17T16:42:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:42:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:42:07Z" name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j.18a73285a0df1eeb namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: 46b38b4b-dad1-442e-a4ee-94623f264fc4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-17T16:42:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 472fed14-7cb5-47c1-8f39-8fbe93f72718 kind: Event lastTimestamp: "2026-04-17T16:43:34Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:42:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:34Z" name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j.18a73285c004cb3d namespace: kserve-ci-e2e-test resourceVersion: "15321" uid: 1736d74d-99d4-4aff-b920-04cc87fca72a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 472fed14-7cb5-47c1-8f39-8fbe93f72718 kind: Event lastTimestamp: "2026-04-17T16:43:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T16:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:33Z" name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j.18a73299b883bbb9 namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: 0b30e0a4-5505-4624-b8ac-370c871dbd4c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 472fed14-7cb5-47c1-8f39-8fbe93f72718 kind: Event lastTimestamp: "2026-04-17T16:43:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:33Z" name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j.18a73299b884f3c0 namespace: kserve-ci-e2e-test resourceVersion: "15304" uid: 497290dd-b21b-4945-b072-7def2fe023b7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 472fed14-7cb5-47c1-8f39-8fbe93f72718 kind: Event lastTimestamp: "2026-04-17T16:43:34Z" message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp 10.134.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:34Z" name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j.18a73299c62c3899 namespace: kserve-ci-e2e-test resourceVersion: "15320" uid: 7d559f17-5823-42c4-95f6-949465f7ad4f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57 namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: c422e66e-e78a-4a56-830d-e2734f61c8c9 kind: Event lastTimestamp: "2026-04-17T16:41:43Z" message: 'Created pod: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57-2b49j' metadata: creationTimestamp: "2026-04-17T16:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:41:43Z" name: isvc-xgboost-graph-raw-f88b3-predictor-55768ff57.18a732800bb05bff namespace: kserve-ci-e2e-test resourceVersion: "14256" uid: b2905743-937b-475f-9f11-5be9e06eced4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-f88b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "14240" uid: 561e88c9-d2d9-4ac7-bcd6-98f4ab6c7a58 kind: Event lastTimestamp: "2026-04-17T16:41:43Z" message: Scaled up replica set isvc-xgboost-graph-raw-f88b3-predictor-55768ff57 from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:41:43Z" name: isvc-xgboost-graph-raw-f88b3-predictor.18a7328008f991c7 namespace: kserve-ci-e2e-test resourceVersion: "14246" uid: f1e51ef7-7cd4-40c0-996a-e58801994676 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:41:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-f88b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "14242" uid: 9c77afe5-c53e-46d7-a0f5-ddb638d2d01b kind: Event lastTimestamp: "2026-04-17T16:42:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:41:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:42:28Z" name: isvc-xgboost-graph-raw-f88b3-predictor.18a732838771efd2 namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: ee11b231-a86d-4e29-87ae-56a75563bbe0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:41:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-f88b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "14242" uid: 9c77afe5-c53e-46d7-a0f5-ddb638d2d01b kind: Event lastTimestamp: "2026-04-17T16:42:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:41:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:42:28Z" name: isvc-xgboost-graph-raw-f88b3-predictor.18a7328387d448bb namespace: kserve-ci-e2e-test resourceVersion: "14663" uid: 954bf5d0-f93b-477a-872b-b764cacc519c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T16:42:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-f88b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "14425" uid: 9c77afe5-c53e-46d7-a0f5-ddb638d2d01b kind: Event lastTimestamp: "2026-04-17T16:43:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:42:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:28Z" name: isvc-xgboost-graph-raw-f88b3-predictor.18a7328e03e76d39 namespace: kserve-ci-e2e-test resourceVersion: "15154" uid: 98abdb98-e585-45d5-a370-30bbf58babd6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T16:42:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-f88b3-predictor namespace: kserve-ci-e2e-test resourceVersion: "14425" uid: 9c77afe5-c53e-46d7-a0f5-ddb638d2d01b kind: Event lastTimestamp: "2026-04-17T16:43:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:42:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:28Z" name: isvc-xgboost-graph-raw-f88b3-predictor.18a7328e0450bdbf namespace: kserve-ci-e2e-test resourceVersion: "15155" uid: 7a1c0e9b-7dcc-4d78-a81c-a87c2943ad2f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-17T16:41:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-f88b3 namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 290a7b45-0fb4-4db5-b999-5257afd690e2 kind: Event lastTimestamp: "2026-04-17T16:43:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:43:14Z" name: isvc-xgboost-graph-raw-f88b3.18a7328008cf5c0f namespace: kserve-ci-e2e-test resourceVersion: "15017" uid: 58ca2517-bb24-47ae-b20a-8b8d8c1722ef reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:41:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-f88b3 namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 290a7b45-0fb4-4db5-b999-5257afd690e2 kind: Event lastTimestamp: "2026-04-17T16:41:43Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-f88b3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-f88b3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:41:43Z" name: isvc-xgboost-graph-raw-f88b3.18a73280098b1bf3 namespace: kserve-ci-e2e-test resourceVersion: "14249" uid: d048edef-5246-4d75-b274-f20c9557dc6d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-f88b3 namespace: kserve-ci-e2e-test resourceVersion: "15015" uid: 290a7b45-0fb4-4db5-b999-5257afd690e2 kind: Event lastTimestamp: "2026-04-17T16:43:14Z" message: InferenceService [isvc-xgboost-graph-raw-f88b3] is Ready metadata: creationTimestamp: "2026-04-17T16:43:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:43:14Z" name: isvc-xgboost-graph-raw-f88b3.18a7329522e3e176 namespace: kserve-ci-e2e-test resourceVersion: "15016" uid: dc8662ef-ed03-4faa-b3aa-48f6ab62ce72 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:43:33.889530Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq namespace: kserve-ci-e2e-test resourceVersion: "15308" uid: f113866f-63b7-45d3-bce0-ac0b5a81c137 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq to ip-10-0-135-127.ec2.internal metadata: creationTimestamp: "2026-04-17T16:43:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:43:33Z" name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq.18a73299bc2f09c9 namespace: kserve-ci-e2e-test resourceVersion: "15313" uid: 054ea162-4ab8-41af-8c41-d070d6aad5a6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f54646664-zzsrx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:34Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq namespace: kserve-ci-e2e-test resourceVersion: "15310" uid: f113866f-63b7-45d3-bce0-ac0b5a81c137 kind: Event lastTimestamp: "2026-04-17T16:43:34Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-hpa-10815-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T16:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:34Z" name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq.18a73299c777a14a namespace: kserve-ci-e2e-test resourceVersion: "15322" uid: 1d196fe0-ccbe-44c2-9b4e-067801d60e9c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:34Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq namespace: kserve-ci-e2e-test resourceVersion: "15319" uid: f113866f-63b7-45d3-bce0-ac0b5a81c137 kind: Event lastTimestamp: "2026-04-17T16:43:34Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:43:34Z" name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq.18a73299f8e4881b namespace: kserve-ci-e2e-test resourceVersion: "15343" uid: f08b57e2-88c2-4046-9534-67844d86d27d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq namespace: kserve-ci-e2e-test resourceVersion: "15310" uid: f113866f-63b7-45d3-bce0-ac0b5a81c137 kind: Event lastTimestamp: "2026-04-17T16:43:34Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0cd196d4c53b891914316f18ab5cfa9f85258e057f3687e65332c70bf642d22d" already present on machine metadata: creationTimestamp: "2026-04-17T16:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:34Z" name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq.18a73299fa521467 namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: 9d801dc8-47d7-4730-9dd6-2403f5753dd8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq namespace: kserve-ci-e2e-test resourceVersion: "15310" uid: f113866f-63b7-45d3-bce0-ac0b5a81c137 kind: Event lastTimestamp: "2026-04-17T16:43:34Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-17T16:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:34Z" name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq.18a73299fdca3981 namespace: kserve-ci-e2e-test resourceVersion: "15346" uid: a40a0e98-b4ec-406a-bc3a-bcc9168c15fc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq namespace: kserve-ci-e2e-test resourceVersion: "15310" uid: f113866f-63b7-45d3-bce0-ac0b5a81c137 kind: Event lastTimestamp: "2026-04-17T16:43:34Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-17T16:43:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:35Z" name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq.18a73299fe3e80d0 namespace: kserve-ci-e2e-test resourceVersion: "15349" uid: 77f18454-4ee8-4a74-bcea-8e7402045d44 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq namespace: kserve-ci-e2e-test resourceVersion: "15310" uid: f113866f-63b7-45d3-bce0-ac0b5a81c137 kind: Event lastTimestamp: "2026-04-17T16:43:39Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-04-17T16:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:39Z" name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq.18a7329b01d37b0a namespace: kserve-ci-e2e-test resourceVersion: "15405" uid: 9230d454-3eb5-4a3f-b6c8-b1d6846a6d7c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq namespace: kserve-ci-e2e-test resourceVersion: "15310" uid: f113866f-63b7-45d3-bce0-ac0b5a81c137 kind: Event lastTimestamp: "2026-04-17T16:43:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:39Z" name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq.18a7329b060094b9 namespace: kserve-ci-e2e-test resourceVersion: "15412" uid: d1342f50-c2fb-4dac-8108-8da308397c59 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq namespace: kserve-ci-e2e-test resourceVersion: "15310" uid: f113866f-63b7-45d3-bce0-ac0b5a81c137 kind: Event lastTimestamp: "2026-04-17T16:43:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:39Z" name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq.18a7329b0678a8a3 namespace: kserve-ci-e2e-test resourceVersion: "15413" uid: 780f0651-2cd3-4ea5-b9af-5e63d46eaec0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq namespace: kserve-ci-e2e-test resourceVersion: "15310" uid: f113866f-63b7-45d3-bce0-ac0b5a81c137 kind: Event lastTimestamp: "2026-04-17T16:43:39Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T16:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:39Z" name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq.18a7329b0685833e namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: 32cce6b5-8842-453d-95f1-f1dd1f34a2d6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq namespace: kserve-ci-e2e-test resourceVersion: "15310" uid: f113866f-63b7-45d3-bce0-ac0b5a81c137 kind: Event lastTimestamp: "2026-04-17T16:43:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:39Z" name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq.18a7329b0987944e namespace: kserve-ci-e2e-test resourceVersion: "15417" uid: 29992aad-d121-41d2-b646-a1e94cd1d3f6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq namespace: kserve-ci-e2e-test resourceVersion: "15310" uid: f113866f-63b7-45d3-bce0-ac0b5a81c137 kind: Event lastTimestamp: "2026-04-17T16:43:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:39Z" name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq.18a7329b09f84847 namespace: kserve-ci-e2e-test resourceVersion: "15418" uid: 5d488f0b-ab63-4a06-869e-9365462e7f96 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-17T16:43:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq namespace: kserve-ci-e2e-test resourceVersion: "15310" uid: f113866f-63b7-45d3-bce0-ac0b5a81c137 kind: Event lastTimestamp: "2026-04-17T16:45:16Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:43:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:16Z" name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq.18a7329b3de91b9b namespace: kserve-ci-e2e-test resourceVersion: "16294" uid: 9b40e1ec-d5eb-475e-b051-ea100c7eb53c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq namespace: kserve-ci-e2e-test resourceVersion: "15310" uid: f113866f-63b7-45d3-bce0-ac0b5a81c137 kind: Event lastTimestamp: "2026-04-17T16:45:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T16:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:14Z" name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq.18a732b10f28e03c namespace: kserve-ci-e2e-test resourceVersion: "16259" uid: dfcf0ca8-971d-4ecf-9fac-4ca3f341c3cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq namespace: kserve-ci-e2e-test resourceVersion: "15310" uid: f113866f-63b7-45d3-bce0-ac0b5a81c137 kind: Event lastTimestamp: "2026-04-17T16:45:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:14Z" name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq.18a732b10f295dcd namespace: kserve-ci-e2e-test resourceVersion: "16260" uid: 87285f57-ba2a-4586-9dee-2e042ff34a0a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq namespace: kserve-ci-e2e-test resourceVersion: "15310" uid: f113866f-63b7-45d3-bce0-ac0b5a81c137 kind: Event lastTimestamp: "2026-04-17T16:45:16Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:16Z" name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq.18a732b19820a72b namespace: kserve-ci-e2e-test resourceVersion: "16293" uid: 23813164-fa09-4d14-b98c-4a590c21f77a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c namespace: kserve-ci-e2e-test resourceVersion: "15273" uid: 9af24f14-783b-4fee-bacc-b34f97bfda7a kind: Event lastTimestamp: "2026-04-17T16:43:33Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c-lzczq' metadata: creationTimestamp: "2026-04-17T16:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:33Z" name: isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c.18a73299bb845550 namespace: kserve-ci-e2e-test resourceVersion: "15309" uid: 9111fb5c-d5be-42dc-87cb-7a45d44bad57 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-10815-predictor namespace: kserve-ci-e2e-test resourceVersion: "15268" uid: bccb6ac2-252f-4d37-b8a0-0a39b2a03a48 kind: Event lastTimestamp: "2026-04-17T16:43:33Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-10815-predictor-74fcc98b7c from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:33Z" name: isvc-xgboost-graph-raw-hpa-10815-predictor.18a73299b4e33e9b namespace: kserve-ci-e2e-test resourceVersion: "15277" uid: b98872fb-d205-4b5d-90f8-f33ef30e041b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:43:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-10815-predictor namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 0568acab-dfcc-46e7-86df-b5d5127ec8f3 kind: Event lastTimestamp: "2026-04-17T16:44:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:43:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:44:03Z" name: isvc-xgboost-graph-raw-hpa-10815-predictor.18a7329d33838464 namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: 9373bdc7-768e-43d4-bbfc-761606326993 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:43:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-10815-predictor namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 0568acab-dfcc-46e7-86df-b5d5127ec8f3 kind: Event lastTimestamp: "2026-04-17T16:44:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:43:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:44:03Z" name: isvc-xgboost-graph-raw-hpa-10815-predictor.18a7329d33e599d4 namespace: kserve-ci-e2e-test resourceVersion: "15612" uid: a14b8b78-c1be-4872-b6ad-ab8d218f743a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T16:44:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-10815-predictor namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: 0568acab-dfcc-46e7-86df-b5d5127ec8f3 kind: Event lastTimestamp: "2026-04-17T16:45:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:44:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:45:03Z" name: isvc-xgboost-graph-raw-hpa-10815-predictor.18a732a431485f64 namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: 8568fc78-b937-4706-8d62-7f544228245b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T16:44:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-10815-predictor namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: 0568acab-dfcc-46e7-86df-b5d5127ec8f3 kind: Event lastTimestamp: "2026-04-17T16:45:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:44:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:45:03Z" name: isvc-xgboost-graph-raw-hpa-10815-predictor.18a732a431b658f2 namespace: kserve-ci-e2e-test resourceVersion: "16091" uid: 53395df8-018d-4f45-9fdd-51d46062a60a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-17T16:43:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-10815 namespace: kserve-ci-e2e-test resourceVersion: "15259" uid: fb6190c1-f06e-434a-8060-933e69885eed kind: Event lastTimestamp: "2026-04-17T16:44:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:44:46Z" name: isvc-xgboost-graph-raw-hpa-10815.18a73299b4e407d2 namespace: kserve-ci-e2e-test resourceVersion: "15933" uid: fbc70784-9e8b-4781-a2d2-5844d4389387 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:44:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-10815 namespace: kserve-ci-e2e-test resourceVersion: "15931" uid: fb6190c1-f06e-434a-8060-933e69885eed kind: Event lastTimestamp: "2026-04-17T16:44:46Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-10815] is Ready metadata: creationTimestamp: "2026-04-17T16:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:44:46Z" name: isvc-xgboost-graph-raw-hpa-10815.18a732aaa0e6ee17 namespace: kserve-ci-e2e-test resourceVersion: "15932" uid: ca99ca69-84c9-44cb-acc3-e133faa06649 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:45:14.018158Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-cd35e-predictor-b88c449b8-w9kjr namespace: kserve-ci-e2e-test resourceVersion: "16229" uid: 3abe04ba-6a27-4d5f-b85d-e7df00cffb69 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-cd35e-predictor-b88c449b8-w9kjr to ip-10-0-135-127.ec2.internal metadata: creationTimestamp: "2026-04-17T16:45:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:45:14Z" name: message-dumper-raw-cd35e-predictor-b88c449b8-w9kjr.18a732b10c50b152 namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: 8afd2f80-c96e-4e1c-9277-119999e4bc8a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f54646664-zzsrx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:14Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-cd35e-predictor-b88c449b8-w9kjr namespace: kserve-ci-e2e-test resourceVersion: "16247" uid: 3abe04ba-6a27-4d5f-b85d-e7df00cffb69 kind: Event lastTimestamp: "2026-04-17T16:45:14Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:45:14Z" name: message-dumper-raw-cd35e-predictor-b88c449b8-w9kjr.18a732b125700d5e namespace: kserve-ci-e2e-test resourceVersion: "16265" uid: 18ad05f4-6e9e-45f4-b25a-42e130508c30 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-cd35e-predictor-b88c449b8-w9kjr namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 3abe04ba-6a27-4d5f-b85d-e7df00cffb69 kind: Event lastTimestamp: "2026-04-17T16:45:14Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-04-17T16:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:14Z" name: message-dumper-raw-cd35e-predictor-b88c449b8-w9kjr.18a732b126ba037f namespace: kserve-ci-e2e-test resourceVersion: "16267" uid: 5082a4ec-b112-4f6f-a62a-47819f7d0c7f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-cd35e-predictor-b88c449b8-w9kjr namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 3abe04ba-6a27-4d5f-b85d-e7df00cffb69 kind: Event lastTimestamp: "2026-04-17T16:45:15Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.332s (1.332s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-04-17T16:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:15Z" name: message-dumper-raw-cd35e-predictor-b88c449b8-w9kjr.18a732b17620a7fc namespace: kserve-ci-e2e-test resourceVersion: "16281" uid: ce284b63-4ac5-41a6-8ca1-d833e578a9bb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-cd35e-predictor-b88c449b8-w9kjr namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 3abe04ba-6a27-4d5f-b85d-e7df00cffb69 kind: Event lastTimestamp: "2026-04-17T16:45:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:15Z" name: message-dumper-raw-cd35e-predictor-b88c449b8-w9kjr.18a732b179dd84b8 namespace: kserve-ci-e2e-test resourceVersion: "16286" uid: 6b62370e-504a-4a63-94fc-dfbb7214174b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-cd35e-predictor-b88c449b8-w9kjr namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 3abe04ba-6a27-4d5f-b85d-e7df00cffb69 kind: Event lastTimestamp: "2026-04-17T16:45:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:15Z" name: message-dumper-raw-cd35e-predictor-b88c449b8-w9kjr.18a732b17a571f9e namespace: kserve-ci-e2e-test resourceVersion: "16287" uid: e048352b-0fa6-4a01-9edf-29c942cb6955 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-cd35e-predictor-b88c449b8-w9kjr namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 3abe04ba-6a27-4d5f-b85d-e7df00cffb69 kind: Event lastTimestamp: "2026-04-17T16:45:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T16:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:15Z" name: message-dumper-raw-cd35e-predictor-b88c449b8-w9kjr.18a732b17a648252 namespace: kserve-ci-e2e-test resourceVersion: "16288" uid: 6e7741f5-0377-411a-9234-bd093d3cbd4f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-cd35e-predictor-b88c449b8-w9kjr namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 3abe04ba-6a27-4d5f-b85d-e7df00cffb69 kind: Event lastTimestamp: "2026-04-17T16:45:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:15Z" name: message-dumper-raw-cd35e-predictor-b88c449b8-w9kjr.18a732b17e4cf4cb namespace: kserve-ci-e2e-test resourceVersion: "16289" uid: 326d9b4f-9b3a-425c-8c7a-b27621da861f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-cd35e-predictor-b88c449b8-w9kjr namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 3abe04ba-6a27-4d5f-b85d-e7df00cffb69 kind: Event lastTimestamp: "2026-04-17T16:45:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:15Z" name: message-dumper-raw-cd35e-predictor-b88c449b8-w9kjr.18a732b17ec029ab namespace: kserve-ci-e2e-test resourceVersion: "16290" uid: 5203520a-0a7e-44c6-969b-ac7fd3b5e621 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:46:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-cd35e-predictor-b88c449b8-w9kjr namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 3abe04ba-6a27-4d5f-b85d-e7df00cffb69 kind: Event lastTimestamp: "2026-04-17T16:46:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T16:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:46:49Z" name: message-dumper-raw-cd35e-predictor-b88c449b8-w9kjr.18a732c73f4b46af namespace: kserve-ci-e2e-test resourceVersion: "17125" uid: 20d6c17b-8a54-4db4-9183-1eaecabb6233 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:46:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-cd35e-predictor-b88c449b8-w9kjr namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 3abe04ba-6a27-4d5f-b85d-e7df00cffb69 kind: Event lastTimestamp: "2026-04-17T16:46:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:46:49Z" name: message-dumper-raw-cd35e-predictor-b88c449b8-w9kjr.18a732c73f4bdbd2 namespace: kserve-ci-e2e-test resourceVersion: "17126" uid: c8ad9843-d770-4189-b51e-3ea001ff98e3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-cd35e-predictor-b88c449b8 namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: cf6e6a40-290b-41e6-9357-d798ae586258 kind: Event lastTimestamp: "2026-04-17T16:45:14Z" message: 'Created pod: message-dumper-raw-cd35e-predictor-b88c449b8-w9kjr' metadata: creationTimestamp: "2026-04-17T16:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:45:14Z" name: message-dumper-raw-cd35e-predictor-b88c449b8.18a732b10bd5602f namespace: kserve-ci-e2e-test resourceVersion: "16232" uid: a1c393ae-3ff3-45cd-913d-7cbf0d37bafa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-cd35e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16211" uid: c1bddbd6-9084-449b-92b6-d7200c4df6b8 kind: Event lastTimestamp: "2026-04-17T16:45:13Z" message: Scaled up replica set message-dumper-raw-cd35e-predictor-b88c449b8 from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:45:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:45:13Z" name: message-dumper-raw-cd35e-predictor.18a732b10884c72d namespace: kserve-ci-e2e-test resourceVersion: "16221" uid: 70b2f06e-fcca-4e4c-8dce-7ce6ab9875bf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:45:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-cd35e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16216" uid: ed9d0da9-55a2-40cd-808e-846acd194507 kind: Event lastTimestamp: "2026-04-17T16:45:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:45:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:45:43Z" name: message-dumper-raw-cd35e-predictor.18a732b4870352c3 namespace: kserve-ci-e2e-test resourceVersion: "16583" uid: fb1e55b0-3525-4509-bcf3-4f435b28b87e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:45:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-cd35e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16216" uid: ed9d0da9-55a2-40cd-808e-846acd194507 kind: Event lastTimestamp: "2026-04-17T16:45:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:45:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:45:43Z" name: message-dumper-raw-cd35e-predictor.18a732b4876025c9 namespace: kserve-ci-e2e-test resourceVersion: "16584" uid: edad7c05-7324-4e71-b780-bf4ec08d9eeb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T16:45:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-cd35e namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: 7609db74-2e7c-4ea2-9d0d-994168c6d22a kind: Event lastTimestamp: "2026-04-17T16:45:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:45:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:45:23Z" name: message-dumper-raw-cd35e.18a732b10862966c namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 79ce4498-e1a6-4f6a-afbd-7e277ac84aad reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-cd35e namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: 7609db74-2e7c-4ea2-9d0d-994168c6d22a kind: Event lastTimestamp: "2026-04-17T16:45:13Z" message: 'Failed to update status for InferenceService "message-dumper-raw-cd35e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-cd35e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:45:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:45:13Z" name: message-dumper-raw-cd35e.18a732b10929ffbf namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: 8db375ec-5938-45fd-931b-ad1ea81661e3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-cd35e namespace: kserve-ci-e2e-test resourceVersion: "16362" uid: 7609db74-2e7c-4ea2-9d0d-994168c6d22a kind: Event lastTimestamp: "2026-04-17T16:45:23Z" message: InferenceService [message-dumper-raw-cd35e] is Ready metadata: creationTimestamp: "2026-04-17T16:45:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:45:23Z" name: message-dumper-raw-cd35e.18a732b350465cec namespace: kserve-ci-e2e-test resourceVersion: "16363" uid: 90ffbdc3-19a8-43e3-8018-67627646789f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:43:23.451244Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-f88b3-6bdd8989c-6mtz9 namespace: kserve-ci-e2e-test resourceVersion: "15089" uid: 3da04f35-cb57-4500-9533-465fb969fc44 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-f88b3-6bdd8989c-6mtz9 to ip-10-0-141-239.ec2.internal metadata: creationTimestamp: "2026-04-17T16:43:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:43:23Z" name: model-chainer-raw-f88b3-6bdd8989c-6mtz9.18a732974e037fcc namespace: kserve-ci-e2e-test resourceVersion: "15096" uid: 540b1c14-191a-4aec-a5bc-32e3ace8225c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f54646664-zzsrx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:23Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-f88b3-6bdd8989c-6mtz9 namespace: kserve-ci-e2e-test resourceVersion: "15100" uid: 3da04f35-cb57-4500-9533-465fb969fc44 kind: Event lastTimestamp: "2026-04-17T16:43:23Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:43:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:43:23Z" name: model-chainer-raw-f88b3-6bdd8989c-6mtz9.18a7329766c8766c namespace: kserve-ci-e2e-test resourceVersion: "15110" uid: a0db3551-05ed-4f47-990d-5b16c011f490 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-f88b3} kind: Pod name: model-chainer-raw-f88b3-6bdd8989c-6mtz9 namespace: kserve-ci-e2e-test resourceVersion: "15093" uid: 3da04f35-cb57-4500-9533-465fb969fc44 kind: Event lastTimestamp: "2026-04-17T16:43:23Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:f452095929b339c4acc2639af4bf7c4693d3269c9b093ed79b1f54101a2de422" metadata: creationTimestamp: "2026-04-17T16:43:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:23Z" name: model-chainer-raw-f88b3-6bdd8989c-6mtz9.18a7329768122859 namespace: kserve-ci-e2e-test resourceVersion: "15112" uid: f1832110-61ec-470f-8b02-5d0c8d2910e5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-239.ec2.internal source: component: kubelet host: ip-10-0-141-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-f88b3} kind: Pod name: model-chainer-raw-f88b3-6bdd8989c-6mtz9 namespace: kserve-ci-e2e-test resourceVersion: "15093" uid: 3da04f35-cb57-4500-9533-465fb969fc44 kind: Event lastTimestamp: "2026-04-17T16:43:25Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:f452095929b339c4acc2639af4bf7c4693d3269c9b093ed79b1f54101a2de422" in 1.748s (1.748s including waiting). Image size: 216233549 bytes.' metadata: creationTimestamp: "2026-04-17T16:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:25Z" name: model-chainer-raw-f88b3-6bdd8989c-6mtz9.18a73297d04faab4 namespace: kserve-ci-e2e-test resourceVersion: "15125" uid: 47567c41-6eec-4f20-b831-c3d102198b71 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-239.ec2.internal source: component: kubelet host: ip-10-0-141-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-f88b3} kind: Pod name: model-chainer-raw-f88b3-6bdd8989c-6mtz9 namespace: kserve-ci-e2e-test resourceVersion: "15093" uid: 3da04f35-cb57-4500-9533-465fb969fc44 kind: Event lastTimestamp: "2026-04-17T16:43:25Z" message: 'Created container: model-chainer-raw-f88b3' metadata: creationTimestamp: "2026-04-17T16:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:25Z" name: model-chainer-raw-f88b3-6bdd8989c-6mtz9.18a73297d39c0a87 namespace: kserve-ci-e2e-test resourceVersion: "15126" uid: 7c6a5571-c0a6-46bb-ad6e-db6e1adce24b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-239.ec2.internal source: component: kubelet host: ip-10-0-141-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-f88b3} kind: Pod name: model-chainer-raw-f88b3-6bdd8989c-6mtz9 namespace: kserve-ci-e2e-test resourceVersion: "15093" uid: 3da04f35-cb57-4500-9533-465fb969fc44 kind: Event lastTimestamp: "2026-04-17T16:43:25Z" message: Started container model-chainer-raw-f88b3 metadata: creationTimestamp: "2026-04-17T16:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:25Z" name: model-chainer-raw-f88b3-6bdd8989c-6mtz9.18a73297d40ef95b namespace: kserve-ci-e2e-test resourceVersion: "15127" uid: 98375fd3-3104-43cd-923c-6398c686e71c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-239.ec2.internal source: component: kubelet host: ip-10-0-141-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-f88b3} kind: Pod name: model-chainer-raw-f88b3-6bdd8989c-6mtz9 namespace: kserve-ci-e2e-test resourceVersion: "15093" uid: 3da04f35-cb57-4500-9533-465fb969fc44 kind: Event lastTimestamp: "2026-04-17T16:43:33Z" message: Stopping container model-chainer-raw-f88b3 metadata: creationTimestamp: "2026-04-17T16:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:43:33Z" name: model-chainer-raw-f88b3-6bdd8989c-6mtz9.18a73299a601b07f namespace: kserve-ci-e2e-test resourceVersion: "15214" uid: fc2dea86-d86c-4731-bab1-bc1ab0492723 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-239.ec2.internal source: component: kubelet host: ip-10-0-141-239.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T16:43:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-f88b3} kind: Pod name: model-chainer-raw-f88b3-6bdd8989c-6mtz9 namespace: kserve-ci-e2e-test resourceVersion: "15093" uid: 3da04f35-cb57-4500-9533-465fb969fc44 kind: Event lastTimestamp: "2026-04-17T16:44:02Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T16:43:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:44:02Z" name: model-chainer-raw-f88b3-6bdd8989c-6mtz9.18a7329a943a9f5c namespace: kserve-ci-e2e-test resourceVersion: "15598" uid: 75b7783f-e3af-4e98-80e6-ef31edce45e7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-239.ec2.internal source: component: kubelet host: ip-10-0-141-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-f88b3-6bdd8989c namespace: kserve-ci-e2e-test resourceVersion: "15086" uid: e60d7001-aa5a-4f2a-8717-39d25fe42b7a kind: Event lastTimestamp: "2026-04-17T16:43:23Z" message: 'Created pod: model-chainer-raw-f88b3-6bdd8989c-6mtz9' metadata: creationTimestamp: "2026-04-17T16:43:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:23Z" name: model-chainer-raw-f88b3-6bdd8989c.18a732974d604859 namespace: kserve-ci-e2e-test resourceVersion: "15092" uid: 05628ea2-afaa-4fc3-85b8-9587a37ec114 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-f88b3 namespace: kserve-ci-e2e-test resourceVersion: "15084" uid: a8854674-9751-408e-9b7b-9eb6e9009e1c kind: Event lastTimestamp: "2026-04-17T16:43:23Z" message: Scaled up replica set model-chainer-raw-f88b3-6bdd8989c from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:43:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:43:23Z" name: model-chainer-raw-f88b3.18a732974c6c85e6 namespace: kserve-ci-e2e-test resourceVersion: "15087" uid: 16400af1-876e-4016-a1a5-e5e5f257bb1b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:43:23Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-f88b3 namespace: kserve-ci-e2e-test resourceVersion: "15102" uid: 21d799d5-0646-4471-b2b1-3f8c5d06087c kind: Event lastTimestamp: "2026-04-17T16:43:23Z" message: InferenceGraph [model-chainer-raw-f88b3] is Ready metadata: creationTimestamp: "2026-04-17T16:43:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:43:23Z" name: model-chainer-raw-f88b3.18a7329750dbd3d0 namespace: kserve-ci-e2e-test resourceVersion: "15104" uid: 8634b17c-be9e-4032-9e93-8f9904fee5ef reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:45:03.777174Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-10815-748448b99c-gzph9 namespace: kserve-ci-e2e-test resourceVersion: "16070" uid: 13b43b45-0905-4c46-9a4a-3be4dd08cb67 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-10815-748448b99c-gzph9 to ip-10-0-141-239.ec2.internal metadata: creationTimestamp: "2026-04-17T16:45:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:45:03Z" name: model-chainer-raw-hpa-10815-748448b99c-gzph9.18a732aea9e7b5eb namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: cdae3213-c974-4169-b8a8-e0930f6a2475 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f54646664-zzsrx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:04Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-10815-748448b99c-gzph9 namespace: kserve-ci-e2e-test resourceVersion: "16082" uid: 13b43b45-0905-4c46-9a4a-3be4dd08cb67 kind: Event lastTimestamp: "2026-04-17T16:45:04Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:45:04Z" name: model-chainer-raw-hpa-10815-748448b99c-gzph9.18a732aec2f0a8ad namespace: kserve-ci-e2e-test resourceVersion: "16093" uid: 67a5a9ae-6957-4176-af56-c48b8b140a12 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-10815} kind: Pod name: model-chainer-raw-hpa-10815-748448b99c-gzph9 namespace: kserve-ci-e2e-test resourceVersion: "16073" uid: 13b43b45-0905-4c46-9a4a-3be4dd08cb67 kind: Event lastTimestamp: "2026-04-17T16:45:04Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f452095929b339c4acc2639af4bf7c4693d3269c9b093ed79b1f54101a2de422" already present on machine metadata: creationTimestamp: "2026-04-17T16:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:04Z" name: model-chainer-raw-hpa-10815-748448b99c-gzph9.18a732aec438bee0 namespace: kserve-ci-e2e-test resourceVersion: "16095" uid: f1eb8624-f26c-497d-b464-77a04d7dcf2c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-239.ec2.internal source: component: kubelet host: ip-10-0-141-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-10815} kind: Pod name: model-chainer-raw-hpa-10815-748448b99c-gzph9 namespace: kserve-ci-e2e-test resourceVersion: "16073" uid: 13b43b45-0905-4c46-9a4a-3be4dd08cb67 kind: Event lastTimestamp: "2026-04-17T16:45:04Z" message: 'Created container: model-chainer-raw-hpa-10815' metadata: creationTimestamp: "2026-04-17T16:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:04Z" name: model-chainer-raw-hpa-10815-748448b99c-gzph9.18a732aec7b5d9fa namespace: kserve-ci-e2e-test resourceVersion: "16097" uid: f76d8685-e80f-401c-b57c-9bcc5c61a058 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-239.ec2.internal source: component: kubelet host: ip-10-0-141-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-10815} kind: Pod name: model-chainer-raw-hpa-10815-748448b99c-gzph9 namespace: kserve-ci-e2e-test resourceVersion: "16073" uid: 13b43b45-0905-4c46-9a4a-3be4dd08cb67 kind: Event lastTimestamp: "2026-04-17T16:45:04Z" message: Started container model-chainer-raw-hpa-10815 metadata: creationTimestamp: "2026-04-17T16:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:04Z" name: model-chainer-raw-hpa-10815-748448b99c-gzph9.18a732aec82a2310 namespace: kserve-ci-e2e-test resourceVersion: "16098" uid: 498ec987-fcd4-4df0-8a87-22f503a719b1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-239.ec2.internal source: component: kubelet host: ip-10-0-141-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-10815} kind: Pod name: model-chainer-raw-hpa-10815-748448b99c-gzph9 namespace: kserve-ci-e2e-test resourceVersion: "16073" uid: 13b43b45-0905-4c46-9a4a-3be4dd08cb67 kind: Event lastTimestamp: "2026-04-17T16:45:13Z" message: Stopping container model-chainer-raw-hpa-10815 metadata: creationTimestamp: "2026-04-17T16:45:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:13Z" name: model-chainer-raw-hpa-10815-748448b99c-gzph9.18a732b1019a52b2 namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: df5ac09c-62d4-426b-a105-9201bce4575e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-239.ec2.internal source: component: kubelet host: ip-10-0-141-239.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T16:45:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-10815} kind: Pod name: model-chainer-raw-hpa-10815-748448b99c-gzph9 namespace: kserve-ci-e2e-test resourceVersion: "16073" uid: 13b43b45-0905-4c46-9a4a-3be4dd08cb67 kind: Event lastTimestamp: "2026-04-17T16:45:40Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T16:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:45:40Z" name: model-chainer-raw-hpa-10815-748448b99c-gzph9.18a732b17809d14d namespace: kserve-ci-e2e-test resourceVersion: "16558" uid: 78664b89-47ee-4fb1-8250-df1da4d20706 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-239.ec2.internal source: component: kubelet host: ip-10-0-141-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-10815-748448b99c namespace: kserve-ci-e2e-test resourceVersion: "16067" uid: 9a1308d3-4d31-4bf7-92c6-22122a65c690 kind: Event lastTimestamp: "2026-04-17T16:45:03Z" message: 'Created pod: model-chainer-raw-hpa-10815-748448b99c-gzph9' metadata: creationTimestamp: "2026-04-17T16:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:45:03Z" name: model-chainer-raw-hpa-10815-748448b99c.18a732aea95cb765 namespace: kserve-ci-e2e-test resourceVersion: "16072" uid: 1f0d671d-ffc0-45ff-99c8-542596af74ac reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-10815 namespace: kserve-ci-e2e-test resourceVersion: "16065" uid: 383d5dd8-db59-4c8b-aafe-4dea63e24c93 kind: Event lastTimestamp: "2026-04-17T16:45:03Z" message: Scaled up replica set model-chainer-raw-hpa-10815-748448b99c from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:45:03Z" name: model-chainer-raw-hpa-10815.18a732aea899ad51 namespace: kserve-ci-e2e-test resourceVersion: "16068" uid: a8126a48-b310-439a-b46b-ca82af03b0a6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:45:03Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-10815 namespace: kserve-ci-e2e-test resourceVersion: "16084" uid: 32da20d0-1994-49b3-a031-2c8893609379 kind: Event lastTimestamp: "2026-04-17T16:45:03Z" message: InferenceGraph [model-chainer-raw-hpa-10815] is Ready metadata: creationTimestamp: "2026-04-17T16:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:45:03Z" name: model-chainer-raw-hpa-10815.18a732aeac66d6d4 namespace: kserve-ci-e2e-test resourceVersion: "16086" uid: a4fe3228-d00b-4e67-8ce8-a5b9ee3e8245 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:51:08.068056Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm namespace: kserve-ci-e2e-test resourceVersion: "19312" uid: a91ab773-e143-4b6d-ab53-9b3b5f1f887a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm to ip-10-0-135-127.ec2.internal metadata: creationTimestamp: "2026-04-17T16:51:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:51:08Z" name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm.18a733037b556e66 namespace: kserve-ci-e2e-test resourceVersion: "19316" uid: 371ddda1-7439-4705-9a40-aa6e95f41476 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f54646664-zzsrx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:51:08Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: a91ab773-e143-4b6d-ab53-9b3b5f1f887a kind: Event lastTimestamp: "2026-04-17T16:51:08Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-466a7-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T16:51:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:51:08Z" name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm.18a73303884cfc0c namespace: kserve-ci-e2e-test resourceVersion: "19328" uid: 277ca0f9-c55f-4414-9fb9-39a98b0af098 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:51:09Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm namespace: kserve-ci-e2e-test resourceVersion: "19321" uid: a91ab773-e143-4b6d-ab53-9b3b5f1f887a kind: Event lastTimestamp: "2026-04-17T16:51:09Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:51:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:51:09Z" name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm.18a73303b83867ac namespace: kserve-ci-e2e-test resourceVersion: "19343" uid: e1881341-8d18-4fb7-b0c5-bba652f69232 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:51:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: a91ab773-e143-4b6d-ab53-9b3b5f1f887a kind: Event lastTimestamp: "2026-04-17T16:51:09Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0cd196d4c53b891914316f18ab5cfa9f85258e057f3687e65332c70bf642d22d" already present on machine metadata: creationTimestamp: "2026-04-17T16:51:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:51:09Z" name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm.18a73303b99d4384 namespace: kserve-ci-e2e-test resourceVersion: "19345" uid: c0db6650-e2d4-4bf8-8afd-08642073e4f5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:51:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: a91ab773-e143-4b6d-ab53-9b3b5f1f887a kind: Event lastTimestamp: "2026-04-17T16:51:09Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-17T16:51:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:51:09Z" name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm.18a73303bc751a70 namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 38c37605-c284-4d62-a853-30cfbfce1eae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:51:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: a91ab773-e143-4b6d-ab53-9b3b5f1f887a kind: Event lastTimestamp: "2026-04-17T16:51:09Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-17T16:51:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:51:09Z" name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm.18a73303bd2c8f63 namespace: kserve-ci-e2e-test resourceVersion: "19347" uid: c6fc9bb0-c034-4479-8fdd-7573e59d1f7d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:51:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: a91ab773-e143-4b6d-ab53-9b3b5f1f887a kind: Event lastTimestamp: "2026-04-17T16:51:12Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T16:51:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:51:12Z" name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm.18a7330494c2f411 namespace: kserve-ci-e2e-test resourceVersion: "19381" uid: 9d55dffb-bd93-4ba9-bff2-1fab6ad6ddab reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:51:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: a91ab773-e143-4b6d-ab53-9b3b5f1f887a kind: Event lastTimestamp: "2026-04-17T16:51:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:51:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:51:12Z" name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm.18a7330497ba3759 namespace: kserve-ci-e2e-test resourceVersion: "19384" uid: 41255b4e-f4a8-4870-97ae-27c6e8eb8532 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:51:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: a91ab773-e143-4b6d-ab53-9b3b5f1f887a kind: Event lastTimestamp: "2026-04-17T16:51:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:51:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:51:12Z" name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm.18a73304982c1dd4 namespace: kserve-ci-e2e-test resourceVersion: "19386" uid: cef493c5-472f-46a1-a679-df07a84cda9c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:51:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: a91ab773-e143-4b6d-ab53-9b3b5f1f887a kind: Event lastTimestamp: "2026-04-17T16:51:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T16:51:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:51:12Z" name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm.18a73304983967ce namespace: kserve-ci-e2e-test resourceVersion: "19387" uid: bf7ed737-c565-4d14-867c-6aef352c5f60 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:51:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: a91ab773-e143-4b6d-ab53-9b3b5f1f887a kind: Event lastTimestamp: "2026-04-17T16:51:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:51:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:51:12Z" name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm.18a733049afa6a1c namespace: kserve-ci-e2e-test resourceVersion: "19389" uid: 0d60fbf6-7232-4f0c-83ef-73a265777a56 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:51:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: a91ab773-e143-4b6d-ab53-9b3b5f1f887a kind: Event lastTimestamp: "2026-04-17T16:51:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:51:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:51:12Z" name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm.18a733049b6eecf8 namespace: kserve-ci-e2e-test resourceVersion: "19390" uid: b0e46a45-6451-4b4e-9704-0c94d26732a8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-17T16:51:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: a91ab773-e143-4b6d-ab53-9b3b5f1f887a kind: Event lastTimestamp: "2026-04-17T16:52:30Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:51:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:30Z" name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm.18a733050c70850e namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 599ca52c-d911-4422-bf3a-117da1c91422 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: a91ab773-e143-4b6d-ab53-9b3b5f1f887a kind: Event lastTimestamp: "2026-04-17T16:52:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T16:52:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:28Z" name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm.18a7331621e6df82 namespace: kserve-ci-e2e-test resourceVersion: "19953" uid: e90d9bd4-3118-417b-9fff-be526ba44499 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: a91ab773-e143-4b6d-ab53-9b3b5f1f887a kind: Event lastTimestamp: "2026-04-17T16:52:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:52:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:28Z" name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm.18a7331621e74e64 namespace: kserve-ci-e2e-test resourceVersion: "19956" uid: c2e2dd25-45da-45ed-b470-25eecefb1723 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: a91ab773-e143-4b6d-ab53-9b3b5f1f887a kind: Event lastTimestamp: "2026-04-17T16:52:30Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:52:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:30Z" name: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm.18a73316beabadb9 namespace: kserve-ci-e2e-test resourceVersion: "20018" uid: bba366d2-a916-485c-a392-deb6b8231b20 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:51:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-466a7-predictor-6cffc9b96c namespace: kserve-ci-e2e-test resourceVersion: "19302" uid: 18586bd8-fa93-495c-a780-062e92d0df2c kind: Event lastTimestamp: "2026-04-17T16:51:08Z" message: 'Created pod: raw-sklearn-466a7-predictor-6cffc9b96c-vq2rm' metadata: creationTimestamp: "2026-04-17T16:51:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:51:08Z" name: raw-sklearn-466a7-predictor-6cffc9b96c.18a733037aee78a9 namespace: kserve-ci-e2e-test resourceVersion: "19313" uid: dd526466-85e7-4954-a963-1abf3e064f92 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:51:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-466a7-predictor namespace: kserve-ci-e2e-test resourceVersion: "19300" uid: 5bef4874-308f-4f91-8222-5b2484847593 kind: Event lastTimestamp: "2026-04-17T16:51:08Z" message: Scaled up replica set raw-sklearn-466a7-predictor-6cffc9b96c from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:51:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:51:08Z" name: raw-sklearn-466a7-predictor.18a733037869b9a3 namespace: kserve-ci-e2e-test resourceVersion: "19304" uid: 24ae141e-a5bb-4755-97bb-3ba872f6feb7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:51:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-466a7-predictor namespace: kserve-ci-e2e-test resourceVersion: "19301" uid: 9b59f69e-159b-4adc-859f-87756f5ffe5c kind: Event lastTimestamp: "2026-04-17T16:51:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:51:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:51:53Z" name: raw-sklearn-466a7-predictor.18a73306f6d9220f namespace: kserve-ci-e2e-test resourceVersion: "19679" uid: 6ede7040-afe7-4700-bb29-d1b78bf6d896 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:51:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-466a7-predictor namespace: kserve-ci-e2e-test resourceVersion: "19301" uid: 9b59f69e-159b-4adc-859f-87756f5ffe5c kind: Event lastTimestamp: "2026-04-17T16:51:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:51:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:51:53Z" name: raw-sklearn-466a7-predictor.18a73306f736a4c5 namespace: kserve-ci-e2e-test resourceVersion: "19680" uid: d03404d7-ff36-4476-8d81-99fed354964e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:52:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-466a7-predictor namespace: kserve-ci-e2e-test resourceVersion: "19468" uid: 9b59f69e-159b-4adc-859f-87756f5ffe5c kind: Event lastTimestamp: "2026-04-17T16:52:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:52:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:52:23Z" name: raw-sklearn-466a7-predictor.18a733117426fca7 namespace: kserve-ci-e2e-test resourceVersion: "19895" uid: 636f60cc-c4ab-49c9-83fc-f0eab8428cb9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:52:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-466a7-predictor namespace: kserve-ci-e2e-test resourceVersion: "19468" uid: 9b59f69e-159b-4adc-859f-87756f5ffe5c kind: Event lastTimestamp: "2026-04-17T16:52:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:52:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:52:23Z" name: raw-sklearn-466a7-predictor.18a73311748a5204 namespace: kserve-ci-e2e-test resourceVersion: "19896" uid: f4e86b38-c1ee-48c5-8555-c8276692dce8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-17T16:51:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-466a7 namespace: kserve-ci-e2e-test resourceVersion: "19289" uid: 9c18be4a-6359-4c96-8bea-3737c658a215 kind: Event lastTimestamp: "2026-04-17T16:52:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:51:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:52:20Z" name: raw-sklearn-466a7.18a7330378436aa3 namespace: kserve-ci-e2e-test resourceVersion: "19880" uid: 841bdbb0-e949-47ac-9f70-241e00f117b8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:51:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-466a7 namespace: kserve-ci-e2e-test resourceVersion: "19289" uid: 9c18be4a-6359-4c96-8bea-3737c658a215 kind: Event lastTimestamp: "2026-04-17T16:51:08Z" message: 'Failed to update status for InferenceService "raw-sklearn-466a7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-466a7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:51:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:51:08Z" name: raw-sklearn-466a7.18a7330378c3e2e1 namespace: kserve-ci-e2e-test resourceVersion: "19333" uid: ce0329b6-61e2-4459-81cb-bb1d15a2e2f9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:51:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-466a7 namespace: kserve-ci-e2e-test resourceVersion: "19310" uid: 9c18be4a-6359-4c96-8bea-3737c658a215 kind: Event lastTimestamp: "2026-04-17T16:51:08Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-466a7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:51:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:51:08Z" name: raw-sklearn-466a7.18a733038cbbd143 namespace: kserve-ci-e2e-test resourceVersion: "19334" uid: 4d9c0752-12f2-4e07-9058-dec33e2d4ff4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-466a7 namespace: kserve-ci-e2e-test resourceVersion: "19877" uid: 9c18be4a-6359-4c96-8bea-3737c658a215 kind: Event lastTimestamp: "2026-04-17T16:52:20Z" message: InferenceService [raw-sklearn-466a7] is Ready metadata: creationTimestamp: "2026-04-17T16:52:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:52:20Z" name: raw-sklearn-466a7.18a733146f8ef540 namespace: kserve-ci-e2e-test resourceVersion: "19879" uid: 157c8bec-ab77-412d-b456-2d930f9f3c72 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T16:52:28.296007Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t namespace: kserve-ci-e2e-test resourceVersion: "19969" uid: 99f446c2-5bd3-45dd-bb5f-6bc80c44195a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t to ip-10-0-135-127.ec2.internal metadata: creationTimestamp: "2026-04-17T16:52:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T16:52:28Z" name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t.18a73316294ad9d3 namespace: kserve-ci-e2e-test resourceVersion: "19976" uid: a6eee00b-ce06-4e1c-a16e-e95a7ecccf21 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7f54646664-zzsrx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:28Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t namespace: kserve-ci-e2e-test resourceVersion: "19980" uid: 99f446c2-5bd3-45dd-bb5f-6bc80c44195a kind: Event lastTimestamp: "2026-04-17T16:52:28Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T16:52:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T16:52:28Z" name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t.18a733164235c816 namespace: kserve-ci-e2e-test resourceVersion: "19991" uid: 1b7f55f4-c1ac-4040-9f01-63c2d3d04a1c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t namespace: kserve-ci-e2e-test resourceVersion: "19974" uid: 99f446c2-5bd3-45dd-bb5f-6bc80c44195a kind: Event lastTimestamp: "2026-04-17T16:52:28Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0cd196d4c53b891914316f18ab5cfa9f85258e057f3687e65332c70bf642d22d" already present on machine metadata: creationTimestamp: "2026-04-17T16:52:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:28Z" name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t.18a7331643872f08 namespace: kserve-ci-e2e-test resourceVersion: "19993" uid: 1854f1e7-52d7-4ee6-9701-bfc121aef4b2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t namespace: kserve-ci-e2e-test resourceVersion: "19974" uid: 99f446c2-5bd3-45dd-bb5f-6bc80c44195a kind: Event lastTimestamp: "2026-04-17T16:52:28Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-17T16:52:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:28Z" name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t.18a7331646272d2b namespace: kserve-ci-e2e-test resourceVersion: "19994" uid: 899e2432-559f-496e-a377-474b5cbe953e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t namespace: kserve-ci-e2e-test resourceVersion: "19974" uid: 99f446c2-5bd3-45dd-bb5f-6bc80c44195a kind: Event lastTimestamp: "2026-04-17T16:52:28Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-17T16:52:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:28Z" name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t.18a733164696abc5 namespace: kserve-ci-e2e-test resourceVersion: "19995" uid: 39ec6be7-d835-41b2-980f-6ce6932449ee reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t namespace: kserve-ci-e2e-test resourceVersion: "19974" uid: 99f446c2-5bd3-45dd-bb5f-6bc80c44195a kind: Event lastTimestamp: "2026-04-17T16:52:33Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T16:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:33Z" name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t.18a7331743916de4 namespace: kserve-ci-e2e-test resourceVersion: "20038" uid: 85f90ec0-34d6-4905-bf07-2c3e62810069 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t namespace: kserve-ci-e2e-test resourceVersion: "19974" uid: 99f446c2-5bd3-45dd-bb5f-6bc80c44195a kind: Event lastTimestamp: "2026-04-17T16:52:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T16:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:33Z" name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t.18a7331746dc4816 namespace: kserve-ci-e2e-test resourceVersion: "20046" uid: 929ff20b-90f3-483d-ad38-53d08044ecd6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t namespace: kserve-ci-e2e-test resourceVersion: "19974" uid: 99f446c2-5bd3-45dd-bb5f-6bc80c44195a kind: Event lastTimestamp: "2026-04-17T16:52:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T16:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:33Z" name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t.18a73317478afe9f namespace: kserve-ci-e2e-test resourceVersion: "20047" uid: f0b1cb90-5a9d-4d32-adaf-2dde19a9e12b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t namespace: kserve-ci-e2e-test resourceVersion: "19974" uid: 99f446c2-5bd3-45dd-bb5f-6bc80c44195a kind: Event lastTimestamp: "2026-04-17T16:52:33Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T16:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:33Z" name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t.18a7331747995d27 namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: 9e28c41e-6b63-44c2-a7ec-5ab6915b3099 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t namespace: kserve-ci-e2e-test resourceVersion: "19974" uid: 99f446c2-5bd3-45dd-bb5f-6bc80c44195a kind: Event lastTimestamp: "2026-04-17T16:52:33Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T16:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:33Z" name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t.18a733174a56895f namespace: kserve-ci-e2e-test resourceVersion: "20050" uid: c2ded38a-6ab6-440c-b35f-6210ef5d1bab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t namespace: kserve-ci-e2e-test resourceVersion: "19974" uid: 99f446c2-5bd3-45dd-bb5f-6bc80c44195a kind: Event lastTimestamp: "2026-04-17T16:52:33Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:52:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:52:33Z" name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t.18a733174ac75234 namespace: kserve-ci-e2e-test resourceVersion: "20051" uid: 617eb1e8-d42f-4161-b028-9432bf4225c3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-17T16:52:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t namespace: kserve-ci-e2e-test resourceVersion: "19974" uid: 99f446c2-5bd3-45dd-bb5f-6bc80c44195a kind: Event lastTimestamp: "2026-04-17T16:53:51Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:52:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:51Z" name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t.18a73317bb67d253 namespace: kserve-ci-e2e-test resourceVersion: "20668" uid: f6e0dec2-e974-4cf8-9036-a70eefa5e352 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t namespace: kserve-ci-e2e-test resourceVersion: "19974" uid: 99f446c2-5bd3-45dd-bb5f-6bc80c44195a kind: Event lastTimestamp: "2026-04-17T16:53:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T16:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:48Z" name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t.18a73328cea41633 namespace: kserve-ci-e2e-test resourceVersion: "20611" uid: c7e4d5f4-43c8-44a2-88a2-69065e0b2549 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t namespace: kserve-ci-e2e-test resourceVersion: "19974" uid: 99f446c2-5bd3-45dd-bb5f-6bc80c44195a kind: Event lastTimestamp: "2026-04-17T16:53:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T16:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:48Z" name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t.18a73328cea4bf28 namespace: kserve-ci-e2e-test resourceVersion: "20612" uid: afe18cea-5eb7-449f-8dd3-6da449a3097a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t namespace: kserve-ci-e2e-test resourceVersion: "19974" uid: 99f446c2-5bd3-45dd-bb5f-6bc80c44195a kind: Event lastTimestamp: "2026-04-17T16:53:51Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp 10.134.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T16:53:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T16:53:51Z" name: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t.18a733296dae9799 namespace: kserve-ci-e2e-test resourceVersion: "20667" uid: a808dee0-5882-43bb-b4c2-eccd250343db reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-127.ec2.internal source: component: kubelet host: ip-10-0-135-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-984d3-predictor-6786bd7bf namespace: kserve-ci-e2e-test resourceVersion: "19960" uid: a806890b-9372-4990-b474-7148f738cabd kind: Event lastTimestamp: "2026-04-17T16:52:28Z" message: 'Created pod: raw-sklearn-runtime-984d3-predictor-6786bd7bf-r9s4t' metadata: creationTimestamp: "2026-04-17T16:52:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:52:28Z" name: raw-sklearn-runtime-984d3-predictor-6786bd7bf.18a733162884d142 namespace: kserve-ci-e2e-test resourceVersion: "19973" uid: 8821ed0e-855d-4fc9-82c3-95eb0b565982 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-984d3-predictor namespace: kserve-ci-e2e-test resourceVersion: "19957" uid: 5ed21bb4-26ca-40a5-ac86-feef6fdc8057 kind: Event lastTimestamp: "2026-04-17T16:52:28Z" message: Scaled up replica set raw-sklearn-runtime-984d3-predictor-6786bd7bf from 0 to 1 metadata: creationTimestamp: "2026-04-17T16:52:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:52:28Z" name: raw-sklearn-runtime-984d3-predictor.18a733162341b89d namespace: kserve-ci-e2e-test resourceVersion: "19962" uid: 39e29ce8-5b03-472d-b74e-701e60181fa1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:52:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-984d3-predictor namespace: kserve-ci-e2e-test resourceVersion: "19959" uid: 41d615af-4f7d-45d3-88bf-adac230a4570 kind: Event lastTimestamp: "2026-04-17T16:52:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:52:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:52:58Z" name: raw-sklearn-runtime-984d3-predictor.18a73319a20ec8bb namespace: kserve-ci-e2e-test resourceVersion: "20228" uid: baa8380d-8e36-43cd-8ab7-6343f7fd0f05 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:52:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-984d3-predictor namespace: kserve-ci-e2e-test resourceVersion: "19959" uid: 41d615af-4f7d-45d3-88bf-adac230a4570 kind: Event lastTimestamp: "2026-04-17T16:52:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T16:52:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:52:58Z" name: raw-sklearn-runtime-984d3-predictor.18a73319a271affd namespace: kserve-ci-e2e-test resourceVersion: "20229" uid: c79f7448-1443-4922-abf9-68379986c5df reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:53:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-984d3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20129" uid: 41d615af-4f7d-45d3-88bf-adac230a4570 kind: Event lastTimestamp: "2026-04-17T16:53:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:53:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:53:43Z" name: raw-sklearn-runtime-984d3-predictor.18a73320a0704cd3 namespace: kserve-ci-e2e-test resourceVersion: "20561" uid: 89bf8117-d946-470c-8f03-1f9a2305db31 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T16:53:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-984d3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20129" uid: 41d615af-4f7d-45d3-88bf-adac230a4570 kind: Event lastTimestamp: "2026-04-17T16:53:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T16:53:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T16:53:43Z" name: raw-sklearn-runtime-984d3-predictor.18a73320a0d6ba96 namespace: kserve-ci-e2e-test resourceVersion: "20562" uid: d881ceae-bac7-498c-abaf-25915fd66d7c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-17T16:52:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-984d3 namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 6e1bb39b-c237-4b7c-82b0-9a5c9929a6d3 kind: Event lastTimestamp: "2026-04-17T16:53:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T16:52:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:53:41Z" name: raw-sklearn-runtime-984d3.18a7331623187894 namespace: kserve-ci-e2e-test resourceVersion: "20545" uid: d28e0726-5ad4-47bd-aa6f-0fecb7bf1acb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T16:52:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-984d3 namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 6e1bb39b-c237-4b7c-82b0-9a5c9929a6d3 kind: Event lastTimestamp: "2026-04-17T16:52:28Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-984d3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-984d3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:52:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:52:28Z" name: raw-sklearn-runtime-984d3.18a73316236fc700 namespace: kserve-ci-e2e-test resourceVersion: "19982" uid: 44ee6c6a-53a7-403c-a898-cef155d6b0a4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:52:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-984d3 namespace: kserve-ci-e2e-test resourceVersion: "19939" uid: 6e1bb39b-c237-4b7c-82b0-9a5c9929a6d3 kind: Event lastTimestamp: "2026-04-17T16:52:28Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-984d3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T16:52:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:52:28Z" name: raw-sklearn-runtime-984d3.18a733162d293651 namespace: kserve-ci-e2e-test resourceVersion: "19983" uid: 9dd09e71-45a5-486c-b101-5c852d1642c5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T16:53:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-984d3 namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: 6e1bb39b-c237-4b7c-82b0-9a5c9929a6d3 kind: Event lastTimestamp: "2026-04-17T16:53:41Z" message: InferenceService [raw-sklearn-runtime-984d3] is Ready metadata: creationTimestamp: "2026-04-17T16:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T16:53:41Z" name: raw-sklearn-runtime-984d3.18a733271eaad488 namespace: kserve-ci-e2e-test resourceVersion: "20543" uid: 8096d6b9-4433-4ff3-b89f-b598e40a5a40 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "20808"