--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:46:00.152189Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-a468f5-predictor-5689d84987-j62j2 namespace: kserve-ci-e2e-test resourceVersion: "19528" uid: b54a3da0-1040-424d-8e05-8b316008c49f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-a468f5-predictor-5689d84987-j62j2 to ip-10-0-132-203.ec2.internal metadata: creationTimestamp: "2026-05-26T11:46:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:46:00Z" name: isvc-init-fail-a468f5-predictor-5689d84987-j62j2.18b31afef98c1203 namespace: kserve-ci-e2e-test resourceVersion: "19532" uid: 75df64bd-ac7d-4950-8dd1-a03f30c6fca8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cfd45b8bd-csfvl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:00Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-a468f5-predictor-5689d84987-j62j2 namespace: kserve-ci-e2e-test resourceVersion: "19530" uid: b54a3da0-1040-424d-8e05-8b316008c49f kind: Event lastTimestamp: "2026-05-26T11:46:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-init-fail-a468f5-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T11:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:46:00Z" name: isvc-init-fail-a468f5-predictor-5689d84987-j62j2.18b31aff045be992 namespace: kserve-ci-e2e-test resourceVersion: "19541" uid: abe24edc-bc65-4897-9044-52eb793bf18b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-a468f5-predictor-5689d84987-j62j2 namespace: kserve-ci-e2e-test resourceVersion: "19536" uid: b54a3da0-1040-424d-8e05-8b316008c49f kind: Event lastTimestamp: "2026-05-26T11:46:01Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:46:01Z" name: isvc-init-fail-a468f5-predictor-5689d84987-j62j2.18b31aff3640c76a namespace: kserve-ci-e2e-test resourceVersion: "19554" uid: 5044077c-104b-4ea3-ac40-123c2bc20fbd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:46:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-a468f5-predictor-5689d84987-j62j2 namespace: kserve-ci-e2e-test resourceVersion: "19530" uid: b54a3da0-1040-424d-8e05-8b316008c49f kind: Event lastTimestamp: "2026-05-26T11:46:05Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:30bffa55f5e3e0e2f6d023c0f84846c206091122d724f5a0e810b4fb957dd389" already present on machine metadata: creationTimestamp: "2026-05-26T11:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:46:05Z" name: isvc-init-fail-a468f5-predictor-5689d84987-j62j2.18b31aff37cdfaa7 namespace: kserve-ci-e2e-test resourceVersion: "19594" uid: d5ecfa04-393f-4c10-b19c-f35e8f3ed3ed reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:46:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-a468f5-predictor-5689d84987-j62j2 namespace: kserve-ci-e2e-test resourceVersion: "19530" uid: b54a3da0-1040-424d-8e05-8b316008c49f kind: Event lastTimestamp: "2026-05-26T11:46:05Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T11:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:46:05Z" name: isvc-init-fail-a468f5-predictor-5689d84987-j62j2.18b31aff3aa604bd namespace: kserve-ci-e2e-test resourceVersion: "19596" uid: 49abffc4-6ba6-482c-b57f-de88f0171135 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:46:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-a468f5-predictor-5689d84987-j62j2 namespace: kserve-ci-e2e-test resourceVersion: "19530" uid: b54a3da0-1040-424d-8e05-8b316008c49f kind: Event lastTimestamp: "2026-05-26T11:46:05Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T11:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:46:05Z" name: isvc-init-fail-a468f5-predictor-5689d84987-j62j2.18b31aff3b127ab4 namespace: kserve-ci-e2e-test resourceVersion: "19599" uid: e5911c1c-5ebe-46b1-a1fa-d1464a75d0f0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-a468f5-predictor-5689d84987-j62j2 namespace: kserve-ci-e2e-test resourceVersion: "19530" uid: b54a3da0-1040-424d-8e05-8b316008c49f kind: Event lastTimestamp: "2026-05-26T11:46:07Z" message: Back-off restarting failed container storage-initializer in pod isvc-init-fail-a468f5-predictor-5689d84987-j62j2_kserve-ci-e2e-test(b54a3da0-1040-424d-8e05-8b316008c49f) metadata: creationTimestamp: "2026-05-26T11:46:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:46:07Z" name: isvc-init-fail-a468f5-predictor-5689d84987-j62j2.18b31b00ab00a4af namespace: kserve-ci-e2e-test resourceVersion: "19618" uid: 0924b37f-abf1-4e73-82f5-f38af1b2960d reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-a468f5-predictor-5689d84987 namespace: kserve-ci-e2e-test resourceVersion: "19519" uid: dad397d0-231d-4961-b7ea-12bd88915dd4 kind: Event lastTimestamp: "2026-05-26T11:46:00Z" message: 'Created pod: isvc-init-fail-a468f5-predictor-5689d84987-j62j2' metadata: creationTimestamp: "2026-05-26T11:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:46:00Z" name: isvc-init-fail-a468f5-predictor-5689d84987.18b31afef921a6cf namespace: kserve-ci-e2e-test resourceVersion: "19529" uid: f5848555-1103-40ac-a944-08a37d26f109 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-a468f5-predictor namespace: kserve-ci-e2e-test resourceVersion: "19517" uid: 39c2cc73-8799-46c4-b5e2-320ce145047d kind: Event lastTimestamp: "2026-05-26T11:46:00Z" message: Scaled up replica set isvc-init-fail-a468f5-predictor-5689d84987 from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:46:00Z" name: isvc-init-fail-a468f5-predictor.18b31afef6043dc9 namespace: kserve-ci-e2e-test resourceVersion: "19521" uid: 97a4dbce-b129-496f-9240-492e3e62e38d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-05-26T11:46:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-a468f5 namespace: kserve-ci-e2e-test resourceVersion: "19489" uid: 3ef20a2f-c561-4a9f-8a83-68417a7aa202 kind: Event lastTimestamp: "2026-05-26T11:46:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:46:08Z" name: isvc-init-fail-a468f5.18b31afef5dd011a namespace: kserve-ci-e2e-test resourceVersion: "19634" uid: b298f67b-75e1-4481-91ca-535f89a7ac59 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-a468f5 namespace: kserve-ci-e2e-test resourceVersion: "19489" uid: 3ef20a2f-c561-4a9f-8a83-68417a7aa202 kind: Event lastTimestamp: "2026-05-26T11:46:00Z" message: 'Failed to update status for InferenceService "isvc-init-fail-a468f5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-a468f5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:46:00Z" name: isvc-init-fail-a468f5.18b31afef631c2d3 namespace: kserve-ci-e2e-test resourceVersion: "19523" uid: 3b51ff08-37e4-487b-a128-a8fa18512dad reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-a468f5 namespace: kserve-ci-e2e-test resourceVersion: "19489" uid: 3ef20a2f-c561-4a9f-8a83-68417a7aa202 kind: Event lastTimestamp: "2026-05-26T11:46:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-a468f5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:46:00Z" name: isvc-init-fail-a468f5.18b31afef6321f01 namespace: kserve-ci-e2e-test resourceVersion: "19524" uid: 1a45ddda-7aed-4fd5-8178-01d2af0a0c50 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:40:40.249214Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq namespace: kserve-ci-e2e-test resourceVersion: "16861" uid: aa97e7de-0a77-4917-b327-35b62f812be4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq to ip-10-0-132-203.ec2.internal metadata: creationTimestamp: "2026-05-26T11:40:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:40:40Z" name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq.18b31ab47dd80f71 namespace: kserve-ci-e2e-test resourceVersion: "16865" uid: a39b4f29-3253-4304-89e3-59844f5d008a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cfd45b8bd-csfvl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:40Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: aa97e7de-0a77-4917-b327-35b62f812be4 kind: Event lastTimestamp: "2026-05-26T11:40:40Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-logger-raw-a937a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T11:40:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:40Z" name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq.18b31ab48a0bc937 namespace: kserve-ci-e2e-test resourceVersion: "16875" uid: cc1bef7a-e746-4d5f-a72f-96608077fb6f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:41Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq namespace: kserve-ci-e2e-test resourceVersion: "16869" uid: aa97e7de-0a77-4917-b327-35b62f812be4 kind: Event lastTimestamp: "2026-05-26T11:40:41Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:40:41Z" name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq.18b31ab4baddfd47 namespace: kserve-ci-e2e-test resourceVersion: "16888" uid: 7f6522c6-f942-4439-88eb-b6d31448505a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: aa97e7de-0a77-4917-b327-35b62f812be4 kind: Event lastTimestamp: "2026-05-26T11:40:41Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:30bffa55f5e3e0e2f6d023c0f84846c206091122d724f5a0e810b4fb957dd389" already present on machine metadata: creationTimestamp: "2026-05-26T11:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:41Z" name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq.18b31ab4bc55298c namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: 42cb539f-24f1-4a6a-b231-b0160c625d24 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: aa97e7de-0a77-4917-b327-35b62f812be4 kind: Event lastTimestamp: "2026-05-26T11:40:41Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T11:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:41Z" name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq.18b31ab4bf1ac5ad namespace: kserve-ci-e2e-test resourceVersion: "16892" uid: 1f933fc6-1fd9-4da9-a63b-dac5d7428e28 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: aa97e7de-0a77-4917-b327-35b62f812be4 kind: Event lastTimestamp: "2026-05-26T11:40:41Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T11:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:41Z" name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq.18b31ab4bf8ae1a1 namespace: kserve-ci-e2e-test resourceVersion: "16893" uid: d353aa34-88de-45a9-98e5-8ca732e42e3c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: aa97e7de-0a77-4917-b327-35b62f812be4 kind: Event lastTimestamp: "2026-05-26T11:40:45Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T11:40:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:45Z" name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq.18b31ab5b6be5eb9 namespace: kserve-ci-e2e-test resourceVersion: "16929" uid: eee63487-2aeb-46e4-90d4-6777952b1113 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: aa97e7de-0a77-4917-b327-35b62f812be4 kind: Event lastTimestamp: "2026-05-26T11:40:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:40:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:45Z" name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq.18b31ab5b987a97e namespace: kserve-ci-e2e-test resourceVersion: "16932" uid: 2f02681e-34dd-4246-acaa-6855fc452e98 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: aa97e7de-0a77-4917-b327-35b62f812be4 kind: Event lastTimestamp: "2026-05-26T11:40:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:40:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:45Z" name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq.18b31ab5b9f7d1d3 namespace: kserve-ci-e2e-test resourceVersion: "16935" uid: df035ff7-74c4-4992-a286-867111bf54ff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: aa97e7de-0a77-4917-b327-35b62f812be4 kind: Event lastTimestamp: "2026-05-26T11:40:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T11:40:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:45Z" name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq.18b31ab5ba03646a namespace: kserve-ci-e2e-test resourceVersion: "16936" uid: da079935-2a42-4904-85fe-acbd374e1fe1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: aa97e7de-0a77-4917-b327-35b62f812be4 kind: Event lastTimestamp: "2026-05-26T11:40:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:40:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:45Z" name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq.18b31ab5bcbcbc50 namespace: kserve-ci-e2e-test resourceVersion: "16938" uid: 363aab7e-856d-488c-b76c-b6cbdf5858e6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: aa97e7de-0a77-4917-b327-35b62f812be4 kind: Event lastTimestamp: "2026-05-26T11:40:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:40:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:45Z" name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq.18b31ab5bd402905 namespace: kserve-ci-e2e-test resourceVersion: "16939" uid: 41824d6d-2d04-43b7-bb8d-25a63ff06d0a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: aa97e7de-0a77-4917-b327-35b62f812be4 kind: Event lastTimestamp: "2026-05-26T11:40:45Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:9d008eea42a91e205ae1ecc0a9946e2c84f08f08aef78aedaed41828bc3d7e9c" already present on machine metadata: creationTimestamp: "2026-05-26T11:40:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:45Z" name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq.18b31ab5bd4ae6e1 namespace: kserve-ci-e2e-test resourceVersion: "16940" uid: 320c673f-a2bc-4b1e-930e-bb760d9909e2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: aa97e7de-0a77-4917-b327-35b62f812be4 kind: Event lastTimestamp: "2026-05-26T11:40:45Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-05-26T11:40:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:45Z" name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq.18b31ab5bff1f063 namespace: kserve-ci-e2e-test resourceVersion: "16941" uid: b4619184-d312-4222-b986-5804c1780fa7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: aa97e7de-0a77-4917-b327-35b62f812be4 kind: Event lastTimestamp: "2026-05-26T11:40:45Z" message: Started container agent metadata: creationTimestamp: "2026-05-26T11:40:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:45Z" name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq.18b31ab5c0656748 namespace: kserve-ci-e2e-test resourceVersion: "16942" uid: 000b3094-ed28-4214-8f0a-9414281a2489 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-26T11:40:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: aa97e7de-0a77-4917-b327-35b62f812be4 kind: Event lastTimestamp: "2026-05-26T11:42:23Z" message: 'Readiness probe failed: dial tcp 10.132.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:40:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:42:23Z" name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq.18b31ab62e688265 namespace: kserve-ci-e2e-test resourceVersion: "17774" uid: 2d83d3a7-d53f-4448-bbac-0af2f3de3bf1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-26T11:40:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: aa97e7de-0a77-4917-b327-35b62f812be4 kind: Event lastTimestamp: "2026-05-26T11:42:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T11:40:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:42:23Z" name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq.18b31ab62e71603c namespace: kserve-ci-e2e-test resourceVersion: "17775" uid: 40bbb74a-0e30-4afc-b8d4-393ad78e5792 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:42:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: aa97e7de-0a77-4917-b327-35b62f812be4 kind: Event lastTimestamp: "2026-05-26T11:42:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T11:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:42:05Z" name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq.18b31ac856e03fb2 namespace: kserve-ci-e2e-test resourceVersion: "17570" uid: 659a65d8-56cf-4d34-9149-ac7b506a70c9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:42:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: aa97e7de-0a77-4917-b327-35b62f812be4 kind: Event lastTimestamp: "2026-05-26T11:42:05Z" message: Stopping container agent metadata: creationTimestamp: "2026-05-26T11:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:42:05Z" name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq.18b31ac856e0ab78 namespace: kserve-ci-e2e-test resourceVersion: "17573" uid: f88dee10-c7d0-4685-9d24-ef2c04a8448b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:42:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: aa97e7de-0a77-4917-b327-35b62f812be4 kind: Event lastTimestamp: "2026-05-26T11:42:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:42:05Z" name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq.18b31ac856e0ed3d namespace: kserve-ci-e2e-test resourceVersion: "17575" uid: 4699aac7-19a9-4a80-aa4c-aab4715aeb82 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T11:42:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: aa97e7de-0a77-4917-b327-35b62f812be4 kind: Event lastTimestamp: "2026-05-26T11:42:23Z" message: 'Readiness probe failed: Get "https://10.132.0.23:8643/healthz": dial tcp 10.132.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:42:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:42:23Z" name: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq.18b31ac90a8a01db namespace: kserve-ci-e2e-test resourceVersion: "17773" uid: bbc7d0a4-439c-45dc-97d0-714fda41fd26 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-a937a-predictor-76c755b66c namespace: kserve-ci-e2e-test resourceVersion: "16850" uid: e0d1ecac-625e-40a3-9604-b01b088c27e7 kind: Event lastTimestamp: "2026-05-26T11:40:40Z" message: 'Created pod: isvc-logger-raw-a937a-predictor-76c755b66c-l7cqq' metadata: creationTimestamp: "2026-05-26T11:40:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:40:40Z" name: isvc-logger-raw-a937a-predictor-76c755b66c.18b31ab47d70085f namespace: kserve-ci-e2e-test resourceVersion: "16862" uid: 3202d850-55d3-4bd7-9069-bcf407fcaa5d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-a937a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16848" uid: b2e1ef8e-e71c-4251-b600-207ff3cf8f63 kind: Event lastTimestamp: "2026-05-26T11:40:40Z" message: Scaled up replica set isvc-logger-raw-a937a-predictor-76c755b66c from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:40:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:40:40Z" name: isvc-logger-raw-a937a-predictor.18b31ab47a516e86 namespace: kserve-ci-e2e-test resourceVersion: "16852" uid: 3e03f3e9-c2a4-428a-b305-98a6f9b3bf41 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-a937a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16849" uid: bc212ce4-b2c3-4eb7-8ca8-dded2e1605fb kind: Event lastTimestamp: "2026-05-26T11:40:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:40:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:40:55Z" name: isvc-logger-raw-a937a-predictor.18b31ab7f8dec35f namespace: kserve-ci-e2e-test resourceVersion: "17021" uid: 1e69a812-4ba1-4907-bb2b-fe8658504b0e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-a937a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16849" uid: bc212ce4-b2c3-4eb7-8ca8-dded2e1605fb kind: Event lastTimestamp: "2026-05-26T11:40:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:40:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:40:55Z" name: isvc-logger-raw-a937a-predictor.18b31ab7f9468330 namespace: kserve-ci-e2e-test resourceVersion: "17023" uid: 26f9f260-69a0-4f05-94dc-cbdb18f87843 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T11:41:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-a937a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17022" uid: bc212ce4-b2c3-4eb7-8ca8-dded2e1605fb kind: Event lastTimestamp: "2026-05-26T11:41:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:41:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:41:55Z" name: isvc-logger-raw-a937a-predictor.18b31abb7879e039 namespace: kserve-ci-e2e-test resourceVersion: "17462" uid: c1966def-c738-443a-92c1-42a4d5babb16 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T11:41:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-a937a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17022" uid: bc212ce4-b2c3-4eb7-8ca8-dded2e1605fb kind: Event lastTimestamp: "2026-05-26T11:41:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:41:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:41:55Z" name: isvc-logger-raw-a937a-predictor.18b31abb78e9a47b namespace: kserve-ci-e2e-test resourceVersion: "17463" uid: 87f0567a-f3b1-4597-94ed-a3e999a08549 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-05-26T11:40:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-a937a namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: c4511d3d-aef5-483f-a5cd-94b9cd2f487a kind: Event lastTimestamp: "2026-05-26T11:41:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:40:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:41:53Z" name: isvc-logger-raw-a937a.18b31ab47a298543 namespace: kserve-ci-e2e-test resourceVersion: "17451" uid: 4e8d3e5d-3b1c-4d54-b172-0c4c31bea13d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:40:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-a937a namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: c4511d3d-aef5-483f-a5cd-94b9cd2f487a kind: Event lastTimestamp: "2026-05-26T11:40:40Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-a937a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-a937a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:40:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:40:40Z" name: isvc-logger-raw-a937a.18b31ab47a834d14 namespace: kserve-ci-e2e-test resourceVersion: "16872" uid: 4f23da1e-72b0-481a-ac86-52c17f8529f9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-a937a namespace: kserve-ci-e2e-test resourceVersion: "16841" uid: c4511d3d-aef5-483f-a5cd-94b9cd2f487a kind: Event lastTimestamp: "2026-05-26T11:40:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-a937a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:40:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:40:40Z" name: isvc-logger-raw-a937a.18b31ab47fa76eda namespace: kserve-ci-e2e-test resourceVersion: "16873" uid: 171411a4-188d-4231-aa3d-67580c603147 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:41:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-a937a namespace: kserve-ci-e2e-test resourceVersion: "17449" uid: c4511d3d-aef5-483f-a5cd-94b9cd2f487a kind: Event lastTimestamp: "2026-05-26T11:41:53Z" message: InferenceService [isvc-logger-raw-a937a] is Ready metadata: creationTimestamp: "2026-05-26T11:41:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:41:53Z" name: isvc-logger-raw-a937a.18b31ac591b4f77a namespace: kserve-ci-e2e-test resourceVersion: "17450" uid: 458b429c-519b-4e7d-b05c-36836151b49d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:44:25.981355Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-287c88-predictor-dbc49469b-lssxz namespace: kserve-ci-e2e-test resourceVersion: "18685" uid: 2822bd4b-a3f3-4452-9575-28b8faf2931c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-287c88-predictor-dbc49469b-lssxz to ip-10-0-132-203.ec2.internal metadata: creationTimestamp: "2026-05-26T11:44:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:44:25Z" name: isvc-primary-287c88-predictor-dbc49469b-lssxz.18b31ae90c872f3d namespace: kserve-ci-e2e-test resourceVersion: "18690" uid: 84e8d4ea-7daf-48be-9afe-7edbe9eb445e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cfd45b8bd-csfvl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:26Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-287c88-predictor-dbc49469b-lssxz namespace: kserve-ci-e2e-test resourceVersion: "18694" uid: 2822bd4b-a3f3-4452-9575-28b8faf2931c kind: Event lastTimestamp: "2026-05-26T11:44:26Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:44:26Z" name: isvc-primary-287c88-predictor-dbc49469b-lssxz.18b31ae9257bc7d0 namespace: kserve-ci-e2e-test resourceVersion: "18700" uid: a20b462f-04a8-4d81-96de-a0b878db2b38 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-287c88-predictor-dbc49469b-lssxz namespace: kserve-ci-e2e-test resourceVersion: "18688" uid: 2822bd4b-a3f3-4452-9575-28b8faf2931c kind: Event lastTimestamp: "2026-05-26T11:44:26Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:30bffa55f5e3e0e2f6d023c0f84846c206091122d724f5a0e810b4fb957dd389" already present on machine metadata: creationTimestamp: "2026-05-26T11:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:26Z" name: isvc-primary-287c88-predictor-dbc49469b-lssxz.18b31ae926e9fcf7 namespace: kserve-ci-e2e-test resourceVersion: "18703" uid: 214a37b0-aaf2-4adb-8eca-512e982a9968 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-287c88-predictor-dbc49469b-lssxz namespace: kserve-ci-e2e-test resourceVersion: "18688" uid: 2822bd4b-a3f3-4452-9575-28b8faf2931c kind: Event lastTimestamp: "2026-05-26T11:44:26Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T11:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:26Z" name: isvc-primary-287c88-predictor-dbc49469b-lssxz.18b31ae929afd9a0 namespace: kserve-ci-e2e-test resourceVersion: "18704" uid: 78740fca-4544-4d0b-9748-8f308e0852bb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-287c88-predictor-dbc49469b-lssxz namespace: kserve-ci-e2e-test resourceVersion: "18688" uid: 2822bd4b-a3f3-4452-9575-28b8faf2931c kind: Event lastTimestamp: "2026-05-26T11:44:26Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T11:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:26Z" name: isvc-primary-287c88-predictor-dbc49469b-lssxz.18b31ae92a1f6f2f namespace: kserve-ci-e2e-test resourceVersion: "18705" uid: 40382215-eeae-4c6b-9670-045ba44a325f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-287c88-predictor-dbc49469b-lssxz namespace: kserve-ci-e2e-test resourceVersion: "18688" uid: 2822bd4b-a3f3-4452-9575-28b8faf2931c kind: Event lastTimestamp: "2026-05-26T11:44:31Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T11:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:31Z" name: isvc-primary-287c88-predictor-dbc49469b-lssxz.18b31aea3ed03a25 namespace: kserve-ci-e2e-test resourceVersion: "18744" uid: 370cf827-81c9-48ea-b18e-0b4d841783c7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-287c88-predictor-dbc49469b-lssxz namespace: kserve-ci-e2e-test resourceVersion: "18688" uid: 2822bd4b-a3f3-4452-9575-28b8faf2931c kind: Event lastTimestamp: "2026-05-26T11:44:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:31Z" name: isvc-primary-287c88-predictor-dbc49469b-lssxz.18b31aea41ef1b56 namespace: kserve-ci-e2e-test resourceVersion: "18746" uid: 4cb64cd8-fe4e-48a0-bb20-bef2ee787c17 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-287c88-predictor-dbc49469b-lssxz namespace: kserve-ci-e2e-test resourceVersion: "18688" uid: 2822bd4b-a3f3-4452-9575-28b8faf2931c kind: Event lastTimestamp: "2026-05-26T11:44:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:31Z" name: isvc-primary-287c88-predictor-dbc49469b-lssxz.18b31aea425d9c54 namespace: kserve-ci-e2e-test resourceVersion: "18748" uid: e3dfcbfd-b2e9-4e8e-9fca-4fce9a397061 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-287c88-predictor-dbc49469b-lssxz namespace: kserve-ci-e2e-test resourceVersion: "18688" uid: 2822bd4b-a3f3-4452-9575-28b8faf2931c kind: Event lastTimestamp: "2026-05-26T11:44:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T11:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:31Z" name: isvc-primary-287c88-predictor-dbc49469b-lssxz.18b31aea42694810 namespace: kserve-ci-e2e-test resourceVersion: "18751" uid: c900cb6c-3f3c-4d31-af91-911a9ca73f83 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-287c88-predictor-dbc49469b-lssxz namespace: kserve-ci-e2e-test resourceVersion: "18688" uid: 2822bd4b-a3f3-4452-9575-28b8faf2931c kind: Event lastTimestamp: "2026-05-26T11:44:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:31Z" name: isvc-primary-287c88-predictor-dbc49469b-lssxz.18b31aea44f98acf namespace: kserve-ci-e2e-test resourceVersion: "18753" uid: aa6364f5-fbe9-40b5-82b0-d248bfd6eafc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-287c88-predictor-dbc49469b-lssxz namespace: kserve-ci-e2e-test resourceVersion: "18688" uid: 2822bd4b-a3f3-4452-9575-28b8faf2931c kind: Event lastTimestamp: "2026-05-26T11:44:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:31Z" name: isvc-primary-287c88-predictor-dbc49469b-lssxz.18b31aea457dd230 namespace: kserve-ci-e2e-test resourceVersion: "18754" uid: 7bdcd83c-0df9-4c84-a1ba-2880020189fc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-26T11:44:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-287c88-predictor-dbc49469b-lssxz namespace: kserve-ci-e2e-test resourceVersion: "18688" uid: 2822bd4b-a3f3-4452-9575-28b8faf2931c kind: Event lastTimestamp: "2026-05-26T11:45:29Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:29Z" name: isvc-primary-287c88-predictor-dbc49469b-lssxz.18b31aeab676ed47 namespace: kserve-ci-e2e-test resourceVersion: "19171" uid: 6b1d3040-550e-4569-9ad3-b2a5ae1a9c7c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-287c88-predictor-dbc49469b-lssxz namespace: kserve-ci-e2e-test resourceVersion: "18688" uid: 2822bd4b-a3f3-4452-9575-28b8faf2931c kind: Event lastTimestamp: "2026-05-26T11:46:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T11:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:46:00Z" name: isvc-primary-287c88-predictor-dbc49469b-lssxz.18b31afef4a8603a namespace: kserve-ci-e2e-test resourceVersion: "19513" uid: ac33fc56-8cca-4610-a031-ce004b0f4d6c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-287c88-predictor-dbc49469b-lssxz namespace: kserve-ci-e2e-test resourceVersion: "18688" uid: 2822bd4b-a3f3-4452-9575-28b8faf2931c kind: Event lastTimestamp: "2026-05-26T11:46:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:46:00Z" name: isvc-primary-287c88-predictor-dbc49469b-lssxz.18b31afef4a910da namespace: kserve-ci-e2e-test resourceVersion: "19516" uid: 46c8949c-cc23-479a-919a-832cc7ad9b50 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-287c88-predictor-dbc49469b namespace: kserve-ci-e2e-test resourceVersion: "18671" uid: 550320d7-3ab5-4fd5-a963-56bc16daf288 kind: Event lastTimestamp: "2026-05-26T11:44:25Z" message: 'Created pod: isvc-primary-287c88-predictor-dbc49469b-lssxz' metadata: creationTimestamp: "2026-05-26T11:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:25Z" name: isvc-primary-287c88-predictor-dbc49469b.18b31ae90bfb47a6 namespace: kserve-ci-e2e-test resourceVersion: "18689" uid: fe9c61d6-2461-4c6d-a3ba-e4f721022d0c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-287c88-predictor namespace: kserve-ci-e2e-test resourceVersion: "18666" uid: a3848dff-0ae4-451f-9ab8-695858cf8b72 kind: Event lastTimestamp: "2026-05-26T11:44:25Z" message: Scaled up replica set isvc-primary-287c88-predictor-dbc49469b from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:25Z" name: isvc-primary-287c88-predictor.18b31ae90216e2df namespace: kserve-ci-e2e-test resourceVersion: "18673" uid: d488b63d-3cc0-4be4-9f8d-dae4c0531abe reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:44:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-287c88-predictor namespace: kserve-ci-e2e-test resourceVersion: "18670" uid: bbfde45f-da7f-4932-af21-8f49e706a48f kind: Event lastTimestamp: "2026-05-26T11:45:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:10Z" name: isvc-primary-287c88-predictor.18b31aec81981af1 namespace: kserve-ci-e2e-test resourceVersion: "19038" uid: cc99d06c-308f-4315-ba0d-8a9cafc83ba3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:44:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-287c88-predictor namespace: kserve-ci-e2e-test resourceVersion: "18670" uid: bbfde45f-da7f-4932-af21-8f49e706a48f kind: Event lastTimestamp: "2026-05-26T11:45:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:10Z" name: isvc-primary-287c88-predictor.18b31aec81fe4d0a namespace: kserve-ci-e2e-test resourceVersion: "19039" uid: 1fb45bb0-c62e-4be3-bc0c-31603caddd2c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:45:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-287c88-predictor namespace: kserve-ci-e2e-test resourceVersion: "18830" uid: bbfde45f-da7f-4932-af21-8f49e706a48f kind: Event lastTimestamp: "2026-05-26T11:45:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:45:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:55Z" name: isvc-primary-287c88-predictor.18b31af6ff6abcc9 namespace: kserve-ci-e2e-test resourceVersion: "19432" uid: f99f44e8-eea8-4668-88a3-dff91d2eca8a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:45:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-287c88-predictor namespace: kserve-ci-e2e-test resourceVersion: "18830" uid: bbfde45f-da7f-4932-af21-8f49e706a48f kind: Event lastTimestamp: "2026-05-26T11:45:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:45:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:55Z" name: isvc-primary-287c88-predictor.18b31af6ffcc8392 namespace: kserve-ci-e2e-test resourceVersion: "19433" uid: a50c4199-3a3f-4042-bc3e-bfd683a52d89 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-05-26T11:44:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-287c88 namespace: kserve-ci-e2e-test resourceVersion: "18640" uid: 08ef98db-e024-4ebc-b51d-ffc2fb49ef47 kind: Event lastTimestamp: "2026-05-26T11:45:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:45:39Z" name: isvc-primary-287c88.18b31ae90215b537 namespace: kserve-ci-e2e-test resourceVersion: "19249" uid: a22fad2a-8f7e-4e51-95b1-3ab8593c99a4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-287c88 namespace: kserve-ci-e2e-test resourceVersion: "18640" uid: 08ef98db-e024-4ebc-b51d-ffc2fb49ef47 kind: Event lastTimestamp: "2026-05-26T11:44:25Z" message: 'Failed to update status for InferenceService "isvc-primary-287c88": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-287c88": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:44:25Z" name: isvc-primary-287c88.18b31ae9029b7345 namespace: kserve-ci-e2e-test resourceVersion: "18675" uid: 7079db57-4132-4be1-814a-5c1218719a40 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-287c88 namespace: kserve-ci-e2e-test resourceVersion: "18640" uid: 08ef98db-e024-4ebc-b51d-ffc2fb49ef47 kind: Event lastTimestamp: "2026-05-26T11:44:25Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-287c88": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:44:25Z" name: isvc-primary-287c88.18b31ae9029bf757 namespace: kserve-ci-e2e-test resourceVersion: "18676" uid: e40a3e0c-1091-417e-b222-237c6bd84187 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-287c88 namespace: kserve-ci-e2e-test resourceVersion: "19246" uid: 08ef98db-e024-4ebc-b51d-ffc2fb49ef47 kind: Event lastTimestamp: "2026-05-26T11:45:39Z" message: InferenceService [isvc-primary-287c88] is Ready metadata: creationTimestamp: "2026-05-26T11:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:45:39Z" name: isvc-primary-287c88.18b31afa19a58d03 namespace: kserve-ci-e2e-test resourceVersion: "19248" uid: d4cca36a-1b58-44d7-9ae0-443398fc01a6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:35:14.401874Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8 namespace: kserve-ci-e2e-test resourceVersion: "13842" uid: 04fe5eed-cfb8-419d-a4c2-d27074305e69 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8 to ip-10-0-132-203.ec2.internal metadata: creationTimestamp: "2026-05-26T11:35:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:35:14Z" name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8.18b31a689fd439b5 namespace: kserve-ci-e2e-test resourceVersion: "13850" uid: d6ca9576-3595-4997-87e4-c73c93c29dd8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cfd45b8bd-csfvl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:35:14Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8 namespace: kserve-ci-e2e-test resourceVersion: "13858" uid: 04fe5eed-cfb8-419d-a4c2-d27074305e69 kind: Event lastTimestamp: "2026-05-26T11:35:14Z" message: Add eth0 [10.132.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:35:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:35:14Z" name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8.18b31a68bd351af6 namespace: kserve-ci-e2e-test resourceVersion: "13867" uid: 63830bdd-aa70-41f0-8091-b69aefa3378e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:35:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8 namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: 04fe5eed-cfb8-419d-a4c2-d27074305e69 kind: Event lastTimestamp: "2026-05-26T11:35:14Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:30bffa55f5e3e0e2f6d023c0f84846c206091122d724f5a0e810b4fb957dd389" metadata: creationTimestamp: "2026-05-26T11:35:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:35:14Z" name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8.18b31a68bf2def8c namespace: kserve-ci-e2e-test resourceVersion: "13869" uid: 1f528cb4-fa3a-4114-9469-5ab40a9633f5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:35:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8 namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: 04fe5eed-cfb8-419d-a4c2-d27074305e69 kind: Event lastTimestamp: "2026-05-26T11:35:19Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:30bffa55f5e3e0e2f6d023c0f84846c206091122d724f5a0e810b4fb957dd389" in 4.138s (4.138s including waiting). Image size: 303109559 bytes.' metadata: creationTimestamp: "2026-05-26T11:35:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:35:19Z" name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8.18b31a69b5d5de54 namespace: kserve-ci-e2e-test resourceVersion: "13901" uid: 65d0fc2c-c7f7-462d-92bf-ef97d2f48a72 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:35:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8 namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: 04fe5eed-cfb8-419d-a4c2-d27074305e69 kind: Event lastTimestamp: "2026-05-26T11:35:19Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T11:35:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:35:19Z" name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8.18b31a69b91b83dc namespace: kserve-ci-e2e-test resourceVersion: "13904" uid: de4c7b1b-55cc-42e2-b9ca-ef19f0624872 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:35:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8 namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: 04fe5eed-cfb8-419d-a4c2-d27074305e69 kind: Event lastTimestamp: "2026-05-26T11:35:19Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T11:35:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:35:19Z" name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8.18b31a69b97515a0 namespace: kserve-ci-e2e-test resourceVersion: "13905" uid: bee99c76-ac59-406e-a160-125eb0eb5f3d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:35:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8 namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: 04fe5eed-cfb8-419d-a4c2-d27074305e69 kind: Event lastTimestamp: "2026-05-26T11:35:23Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" metadata: creationTimestamp: "2026-05-26T11:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:35:23Z" name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8.18b31a6ac1bda069 namespace: kserve-ci-e2e-test resourceVersion: "13939" uid: b445fe18-6923-4252-b1a6-9d042b5d2d27 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:35:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8 namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: 04fe5eed-cfb8-419d-a4c2-d27074305e69 kind: Event lastTimestamp: "2026-05-26T11:35:36Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" in 12.73s (12.73s including waiting). Image size: 1560928695 bytes.' metadata: creationTimestamp: "2026-05-26T11:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:35:36Z" name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8.18b31a6db89182b9 namespace: kserve-ci-e2e-test resourceVersion: "14039" uid: b1a6d28e-fe23-49bb-9d75-7ef5c7d96742 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:35:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8 namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: 04fe5eed-cfb8-419d-a4c2-d27074305e69 kind: Event lastTimestamp: "2026-05-26T11:35:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:35:36Z" name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8.18b31a6dbbbfced3 namespace: kserve-ci-e2e-test resourceVersion: "14041" uid: 79535a7c-d376-4c94-8e22-f60b25a1bec5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:35:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8 namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: 04fe5eed-cfb8-419d-a4c2-d27074305e69 kind: Event lastTimestamp: "2026-05-26T11:35:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:35:36Z" name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8.18b31a6dbc1cffdd namespace: kserve-ci-e2e-test resourceVersion: "14042" uid: 87b23f1f-9fa0-4241-be49-28efb0b7039a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:35:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8 namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: 04fe5eed-cfb8-419d-a4c2-d27074305e69 kind: Event lastTimestamp: "2026-05-26T11:35:36Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-26T11:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:35:36Z" name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8.18b31a6dbc2a7f3a namespace: kserve-ci-e2e-test resourceVersion: "14043" uid: 37c4a162-5026-47fb-b3f9-9f616a6484f8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:35:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8 namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: 04fe5eed-cfb8-419d-a4c2-d27074305e69 kind: Event lastTimestamp: "2026-05-26T11:35:38Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.376s (2.376s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-26T11:35:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:35:38Z" name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8.18b31a6e49d7fde6 namespace: kserve-ci-e2e-test resourceVersion: "14059" uid: abbfca48-b5de-4499-ab05-3895112cdb5e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:35:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8 namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: 04fe5eed-cfb8-419d-a4c2-d27074305e69 kind: Event lastTimestamp: "2026-05-26T11:35:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:35:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:35:38Z" name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8.18b31a6e4cc3ff00 namespace: kserve-ci-e2e-test resourceVersion: "14060" uid: de38ab6f-a023-41ba-bc95-1185162b0419 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:35:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8 namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: 04fe5eed-cfb8-419d-a4c2-d27074305e69 kind: Event lastTimestamp: "2026-05-26T11:35:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:35:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:35:38Z" name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8.18b31a6e4d233533 namespace: kserve-ci-e2e-test resourceVersion: "14061" uid: ab097956-8ffa-42b2-bdb4-fd6261d5eb56 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:35:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8 namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: 04fe5eed-cfb8-419d-a4c2-d27074305e69 kind: Event lastTimestamp: "2026-05-26T11:35:38Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:9d008eea42a91e205ae1ecc0a9946e2c84f08f08aef78aedaed41828bc3d7e9c" metadata: creationTimestamp: "2026-05-26T11:35:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:35:38Z" name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8.18b31a6e4d2fb19e namespace: kserve-ci-e2e-test resourceVersion: "14062" uid: 2ae0ba6d-35dc-472b-a65a-f415be5cc61c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8 namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: 04fe5eed-cfb8-419d-a4c2-d27074305e69 kind: Event lastTimestamp: "2026-05-26T11:35:41Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:9d008eea42a91e205ae1ecc0a9946e2c84f08f08aef78aedaed41828bc3d7e9c" in 2.791s (2.791s including waiting). Image size: 239690329 bytes.' metadata: creationTimestamp: "2026-05-26T11:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:35:41Z" name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8.18b31a6ef3901f35 namespace: kserve-ci-e2e-test resourceVersion: "14079" uid: 41e05db4-0950-4476-acd7-7d8611254606 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8 namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: 04fe5eed-cfb8-419d-a4c2-d27074305e69 kind: Event lastTimestamp: "2026-05-26T11:35:41Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-05-26T11:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:35:41Z" name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8.18b31a6ef644c6dc namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: 529c6104-9868-40f5-bd74-67e6e85d72bc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:35:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8 namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: 04fe5eed-cfb8-419d-a4c2-d27074305e69 kind: Event lastTimestamp: "2026-05-26T11:35:41Z" message: Started container agent metadata: creationTimestamp: "2026-05-26T11:35:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:35:41Z" name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8.18b31a6ef6c60ad7 namespace: kserve-ci-e2e-test resourceVersion: "14081" uid: 1dc32b2a-7363-4fec-ac5f-d9681855df87 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-26T11:35:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8 namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: 04fe5eed-cfb8-419d-a4c2-d27074305e69 kind: Event lastTimestamp: "2026-05-26T11:37:15Z" message: 'Readiness probe failed: dial tcp 10.132.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:35:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:37:15Z" name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8.18b31a6f6dcd0abc namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: 1c8e1286-b88a-4b25-b93f-f55bf7b5966b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-26T11:35:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8 namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: 04fe5eed-cfb8-419d-a4c2-d27074305e69 kind: Event lastTimestamp: "2026-05-26T11:37:15Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T11:35:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:37:15Z" name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8.18b31a6f6dd6c27e namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: 60302c18-d863-49fd-ab3c-d17ebd793b7a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:36:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8 namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: 04fe5eed-cfb8-419d-a4c2-d27074305e69 kind: Event lastTimestamp: "2026-05-26T11:36:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T11:36:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:36:59Z" name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8.18b31a8117cec3a7 namespace: kserve-ci-e2e-test resourceVersion: "14681" uid: b51570f8-82f8-4329-92d7-8da5a601f7cd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:36:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8 namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: 04fe5eed-cfb8-419d-a4c2-d27074305e69 kind: Event lastTimestamp: "2026-05-26T11:36:59Z" message: Stopping container agent metadata: creationTimestamp: "2026-05-26T11:36:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:36:59Z" name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8.18b31a8117cef393 namespace: kserve-ci-e2e-test resourceVersion: "14685" uid: 3c094cf4-bfa7-4bef-8d21-06874d23850c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:36:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8 namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: 04fe5eed-cfb8-419d-a4c2-d27074305e69 kind: Event lastTimestamp: "2026-05-26T11:36:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:36:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:36:59Z" name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8.18b31a8117cf281b namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: cf783f48-1346-470c-b824-e5ef5531f186 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T11:37:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8 namespace: kserve-ci-e2e-test resourceVersion: "13847" uid: 04fe5eed-cfb8-419d-a4c2-d27074305e69 kind: Event lastTimestamp: "2026-05-26T11:37:23Z" message: 'Readiness probe failed: Get "https://10.132.0.17:8643/healthz": dial tcp 10.132.0.17:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:37:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:37:23Z" name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8.18b31a820e2149e3 namespace: kserve-ci-e2e-test resourceVersion: "14965" uid: 3d221a64-240c-4c41-8dd9-6a7aa685117c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:35:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf namespace: kserve-ci-e2e-test resourceVersion: "13831" uid: 2ed30985-a70a-4e88-80e8-506f4e343c57 kind: Event lastTimestamp: "2026-05-26T11:35:14Z" message: 'Created pod: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf-vdbh8' metadata: creationTimestamp: "2026-05-26T11:35:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:35:14Z" name: isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf.18b31a689ef31de5 namespace: kserve-ci-e2e-test resourceVersion: "13845" uid: ac535314-236e-4eb0-ba2b-39e2537b6d3b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:35:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-88958-predictor namespace: kserve-ci-e2e-test resourceVersion: "13830" uid: 2a1d3839-95ec-4b47-8952-44aab0e4fcc0 kind: Event lastTimestamp: "2026-05-26T11:35:14Z" message: Scaled up replica set isvc-raw-sklearn-batcher-88958-predictor-84f8f466bf from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:35:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:35:14Z" name: isvc-raw-sklearn-batcher-88958-predictor.18b31a6897d50ccc namespace: kserve-ci-e2e-test resourceVersion: "13832" uid: b2577a5c-973b-4a93-b613-50b8282928ee reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:35:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-88958-predictor namespace: kserve-ci-e2e-test resourceVersion: "13840" uid: d6a0606b-8f85-4075-bb49-53fd117fee19 kind: Event lastTimestamp: "2026-05-26T11:36:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:35:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:36:14Z" name: isvc-raw-sklearn-batcher-88958-predictor.18b31a6c1d68cd83 namespace: kserve-ci-e2e-test resourceVersion: "14322" uid: bbd3f4b4-6035-482c-93d5-1252cff0d115 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:35:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-88958-predictor namespace: kserve-ci-e2e-test resourceVersion: "13840" uid: d6a0606b-8f85-4075-bb49-53fd117fee19 kind: Event lastTimestamp: "2026-05-26T11:36:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:35:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:36:14Z" name: isvc-raw-sklearn-batcher-88958-predictor.18b31a6c1dce962b namespace: kserve-ci-e2e-test resourceVersion: "14324" uid: 3abc6bbc-1de9-46fa-bcef-5f3d502ceea7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:35:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-88958-predictor namespace: kserve-ci-e2e-test resourceVersion: "13986" uid: d6a0606b-8f85-4075-bb49-53fd117fee19 kind: Event lastTimestamp: "2026-05-26T11:36:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:35:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:36:44Z" name: isvc-raw-sklearn-batcher-88958-predictor.18b31a731c0f5170 namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: 2b0fac82-66c5-47ca-a351-d936c5983d71 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:35:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-88958-predictor namespace: kserve-ci-e2e-test resourceVersion: "13986" uid: d6a0606b-8f85-4075-bb49-53fd117fee19 kind: Event lastTimestamp: "2026-05-26T11:36:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:35:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:36:44Z" name: isvc-raw-sklearn-batcher-88958-predictor.18b31a731cf02d3d namespace: kserve-ci-e2e-test resourceVersion: "14542" uid: f18549eb-5e28-43dc-b407-984d60ce4fba reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 14 eventTime: null firstTimestamp: "2026-05-26T11:35:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-88958 namespace: kserve-ci-e2e-test resourceVersion: "13821" uid: dfc60f17-2e6b-42ad-acce-dec8ad1a4ea6 kind: Event lastTimestamp: "2026-05-26T11:36:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:35:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:36:45Z" name: isvc-raw-sklearn-batcher-88958.18b31a689e0f717c namespace: kserve-ci-e2e-test resourceVersion: "14561" uid: c7b9a16c-5955-40e9-8d49-ca8f34349ce4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:35:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-88958 namespace: kserve-ci-e2e-test resourceVersion: "13821" uid: dfc60f17-2e6b-42ad-acce-dec8ad1a4ea6 kind: Event lastTimestamp: "2026-05-26T11:35:14Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-88958": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-88958": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:35:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:35:14Z" name: isvc-raw-sklearn-batcher-88958.18b31a689ea3435d namespace: kserve-ci-e2e-test resourceVersion: "13843" uid: 0861697e-bb9a-4700-9717-ed86a1e1f139 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:35:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-88958 namespace: kserve-ci-e2e-test resourceVersion: "13821" uid: dfc60f17-2e6b-42ad-acce-dec8ad1a4ea6 kind: Event lastTimestamp: "2026-05-26T11:35:14Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-88958": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:35:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:35:14Z" name: isvc-raw-sklearn-batcher-88958.18b31a689ea3b4ee namespace: kserve-ci-e2e-test resourceVersion: "13846" uid: 001dc071-68c7-4696-a7c5-f6cf6beb5532 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:36:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-88958 namespace: kserve-ci-e2e-test resourceVersion: "14559" uid: dfc60f17-2e6b-42ad-acce-dec8ad1a4ea6 kind: Event lastTimestamp: "2026-05-26T11:36:45Z" message: InferenceService [isvc-raw-sklearn-batcher-88958] is Ready metadata: creationTimestamp: "2026-05-26T11:36:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:36:45Z" name: isvc-raw-sklearn-batcher-88958.18b31a7de448c530 namespace: kserve-ci-e2e-test resourceVersion: "14560" uid: 3ae017a7-e2ca-4f0c-a15b-f079ec0a3ab9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:45:45.975040Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-287c88-predictor-f85b56fbc-fg5zv namespace: kserve-ci-e2e-test resourceVersion: "19321" uid: 70bc9027-29b0-4a3d-9024-15bea0b0442b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-287c88-predictor-f85b56fbc-fg5zv to ip-10-0-132-203.ec2.internal metadata: creationTimestamp: "2026-05-26T11:45:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:45:45Z" name: isvc-secondary-287c88-predictor-f85b56fbc-fg5zv.18b31afbac85deb4 namespace: kserve-ci-e2e-test resourceVersion: "19325" uid: 595cecb5-2899-4cf9-8b36-fb14366eb078 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cfd45b8bd-csfvl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:46Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-287c88-predictor-f85b56fbc-fg5zv namespace: kserve-ci-e2e-test resourceVersion: "19329" uid: 70bc9027-29b0-4a3d-9024-15bea0b0442b kind: Event lastTimestamp: "2026-05-26T11:45:46Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:45:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:45:46Z" name: isvc-secondary-287c88-predictor-f85b56fbc-fg5zv.18b31afbc57ba20f namespace: kserve-ci-e2e-test resourceVersion: "19338" uid: 2e5f100c-4d72-4291-97ba-692c8cee018b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:45:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-287c88-predictor-f85b56fbc-fg5zv namespace: kserve-ci-e2e-test resourceVersion: "19323" uid: 70bc9027-29b0-4a3d-9024-15bea0b0442b kind: Event lastTimestamp: "2026-05-26T11:45:48Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:30bffa55f5e3e0e2f6d023c0f84846c206091122d724f5a0e810b4fb957dd389" already present on machine metadata: creationTimestamp: "2026-05-26T11:45:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:48Z" name: isvc-secondary-287c88-predictor-f85b56fbc-fg5zv.18b31afbc6d5beb2 namespace: kserve-ci-e2e-test resourceVersion: "19369" uid: b665188b-2966-405f-b58b-bcd4328a699a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:45:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-287c88-predictor-f85b56fbc-fg5zv namespace: kserve-ci-e2e-test resourceVersion: "19323" uid: 70bc9027-29b0-4a3d-9024-15bea0b0442b kind: Event lastTimestamp: "2026-05-26T11:45:48Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T11:45:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:48Z" name: isvc-secondary-287c88-predictor-f85b56fbc-fg5zv.18b31afbc97112b5 namespace: kserve-ci-e2e-test resourceVersion: "19373" uid: db62811e-47e3-4217-8b7d-605efe440d47 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:45:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-287c88-predictor-f85b56fbc-fg5zv namespace: kserve-ci-e2e-test resourceVersion: "19323" uid: 70bc9027-29b0-4a3d-9024-15bea0b0442b kind: Event lastTimestamp: "2026-05-26T11:45:48Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T11:45:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:48Z" name: isvc-secondary-287c88-predictor-f85b56fbc-fg5zv.18b31afbc9dd3d6a namespace: kserve-ci-e2e-test resourceVersion: "19374" uid: 14c3b92e-30ef-4306-b6cc-f4ba65701fd3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-287c88-predictor-f85b56fbc-fg5zv namespace: kserve-ci-e2e-test resourceVersion: "19323" uid: 70bc9027-29b0-4a3d-9024-15bea0b0442b kind: Event lastTimestamp: "2026-05-26T11:45:51Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-287c88-predictor-f85b56fbc-fg5zv_kserve-ci-e2e-test(70bc9027-29b0-4a3d-9024-15bea0b0442b) metadata: creationTimestamp: "2026-05-26T11:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:51Z" name: isvc-secondary-287c88-predictor-f85b56fbc-fg5zv.18b31afcb1b40100 namespace: kserve-ci-e2e-test resourceVersion: "19401" uid: 40a9ed09-c399-40cc-9704-7789ccbe8fb1 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-287c88-predictor-f85b56fbc namespace: kserve-ci-e2e-test resourceVersion: "19312" uid: d43721c0-1d33-4d9c-b593-58482f9fa1da kind: Event lastTimestamp: "2026-05-26T11:45:45Z" message: 'Created pod: isvc-secondary-287c88-predictor-f85b56fbc-fg5zv' metadata: creationTimestamp: "2026-05-26T11:45:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:45Z" name: isvc-secondary-287c88-predictor-f85b56fbc.18b31afbabf43fc0 namespace: kserve-ci-e2e-test resourceVersion: "19322" uid: 12d316de-2073-4cd4-b3cb-281dabd315ff reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-287c88-predictor namespace: kserve-ci-e2e-test resourceVersion: "19309" uid: 1e2a3bc2-37cb-4864-b452-7a1dd7c0b613 kind: Event lastTimestamp: "2026-05-26T11:45:45Z" message: Scaled up replica set isvc-secondary-287c88-predictor-f85b56fbc from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:45:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:45Z" name: isvc-secondary-287c88-predictor.18b31afba8d4da97 namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: 1d7a7df2-17b8-4d79-8c4a-c51197d7dd58 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 14 eventTime: null firstTimestamp: "2026-05-26T11:45:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-287c88 namespace: kserve-ci-e2e-test resourceVersion: "19299" uid: 0b84f0f4-b9b7-440f-8746-bd49c1b93058 kind: Event lastTimestamp: "2026-05-26T11:45:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:45:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:45:51Z" name: isvc-secondary-287c88.18b31afba8afd6e8 namespace: kserve-ci-e2e-test resourceVersion: "19406" uid: 15ce3829-a489-4c33-8046-3d0411a04668 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-287c88 namespace: kserve-ci-e2e-test resourceVersion: "19299" uid: 0b84f0f4-b9b7-440f-8746-bd49c1b93058 kind: Event lastTimestamp: "2026-05-26T11:45:45Z" message: 'Failed to update status for InferenceService "isvc-secondary-287c88": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-287c88": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:45:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:45:45Z" name: isvc-secondary-287c88.18b31afba9230625 namespace: kserve-ci-e2e-test resourceVersion: "19316" uid: 3d2e4aef-3d21-410c-ad04-f22abcfea0dd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-287c88 namespace: kserve-ci-e2e-test resourceVersion: "19299" uid: 0b84f0f4-b9b7-440f-8746-bd49c1b93058 kind: Event lastTimestamp: "2026-05-26T11:45:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-287c88": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:45:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:45:45Z" name: isvc-secondary-287c88.18b31afba92356d8 namespace: kserve-ci-e2e-test resourceVersion: "19317" uid: 1961d6fb-8d1d-4e0f-9670-bb5b0cdd020f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:36:59.639692Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n namespace: kserve-ci-e2e-test resourceVersion: "14705" uid: a0687fd1-5202-43b7-af26-951a9fdf49c2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n to ip-10-0-132-203.ec2.internal metadata: creationTimestamp: "2026-05-26T11:36:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:36:59Z" name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n.18b31a81207dd824 namespace: kserve-ci-e2e-test resourceVersion: "14713" uid: c464801a-70a1-49db-b36e-dafdec0debb7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cfd45b8bd-csfvl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:37:00Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n namespace: kserve-ci-e2e-test resourceVersion: "14722" uid: a0687fd1-5202-43b7-af26-951a9fdf49c2 kind: Event lastTimestamp: "2026-05-26T11:37:00Z" message: Add eth0 [10.132.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:37:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:37:00Z" name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n.18b31a8139a34bf7 namespace: kserve-ci-e2e-test resourceVersion: "14749" uid: b41aa954-b99a-4ae2-a6a3-41d0d91218e1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:37:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: a0687fd1-5202-43b7-af26-951a9fdf49c2 kind: Event lastTimestamp: "2026-05-26T11:37:00Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:30bffa55f5e3e0e2f6d023c0f84846c206091122d724f5a0e810b4fb957dd389" already present on machine metadata: creationTimestamp: "2026-05-26T11:37:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:37:00Z" name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n.18b31a813b35684b namespace: kserve-ci-e2e-test resourceVersion: "14751" uid: 5765d119-3488-4f45-aa28-fb84032584ba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:37:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: a0687fd1-5202-43b7-af26-951a9fdf49c2 kind: Event lastTimestamp: "2026-05-26T11:37:00Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T11:37:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:37:00Z" name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n.18b31a813da98725 namespace: kserve-ci-e2e-test resourceVersion: "14752" uid: e39ee9d2-0789-4f5e-a551-549256e67473 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:37:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: a0687fd1-5202-43b7-af26-951a9fdf49c2 kind: Event lastTimestamp: "2026-05-26T11:37:00Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T11:37:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:37:00Z" name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n.18b31a813e030ded namespace: kserve-ci-e2e-test resourceVersion: "14753" uid: 8a2ca923-2c68-473e-bd0c-f21edccb76e1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:37:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: a0687fd1-5202-43b7-af26-951a9fdf49c2 kind: Event lastTimestamp: "2026-05-26T11:37:04Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T11:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:37:04Z" name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n.18b31a8256f8386f namespace: kserve-ci-e2e-test resourceVersion: "14807" uid: aab95229-2d32-4bf9-9139-dcfccda027f8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:37:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: a0687fd1-5202-43b7-af26-951a9fdf49c2 kind: Event lastTimestamp: "2026-05-26T11:37:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:37:04Z" name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n.18b31a825a27976e namespace: kserve-ci-e2e-test resourceVersion: "14810" uid: eb495e15-4150-4217-9f15-68bb1adc6144 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:37:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: a0687fd1-5202-43b7-af26-951a9fdf49c2 kind: Event lastTimestamp: "2026-05-26T11:37:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:37:04Z" name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n.18b31a825a8c1414 namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: c5cad3dd-d4ef-43d4-a01b-37e1c36d2ab0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:37:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: a0687fd1-5202-43b7-af26-951a9fdf49c2 kind: Event lastTimestamp: "2026-05-26T11:37:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T11:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:37:04Z" name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n.18b31a825a992146 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: 51352d2a-36d7-4ce2-8814-42d7cb89fb43 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:37:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: a0687fd1-5202-43b7-af26-951a9fdf49c2 kind: Event lastTimestamp: "2026-05-26T11:37:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:37:04Z" name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n.18b31a825d4aad22 namespace: kserve-ci-e2e-test resourceVersion: "14817" uid: 7e868c0e-d350-42a4-897f-fe524d579daa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:37:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: a0687fd1-5202-43b7-af26-951a9fdf49c2 kind: Event lastTimestamp: "2026-05-26T11:37:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:37:04Z" name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n.18b31a825dabd340 namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 7f7376b7-9dc2-4dfc-b7c8-7b3697ab33ff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-26T11:37:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: a0687fd1-5202-43b7-af26-951a9fdf49c2 kind: Event lastTimestamp: "2026-05-26T11:38:42Z" message: 'Readiness probe failed: dial tcp 10.132.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:42Z" name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n.18b31a82ced318de namespace: kserve-ci-e2e-test resourceVersion: "15767" uid: b9511e16-0ef9-4878-9eaf-c8824003cc0e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: a0687fd1-5202-43b7-af26-951a9fdf49c2 kind: Event lastTimestamp: "2026-05-26T11:38:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T11:38:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:39Z" name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n.18b31a9872706b28 namespace: kserve-ci-e2e-test resourceVersion: "15651" uid: 38ea0cff-368f-488f-9385-1e9bfd86f9fa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: a0687fd1-5202-43b7-af26-951a9fdf49c2 kind: Event lastTimestamp: "2026-05-26T11:38:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:38:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:39Z" name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n.18b31a9872711220 namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: 93e98b00-d217-45f2-a12d-535c815937fc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: a0687fd1-5202-43b7-af26-951a9fdf49c2 kind: Event lastTimestamp: "2026-05-26T11:38:42Z" message: 'Readiness probe failed: Get "https://10.132.0.18:8643/healthz": dial tcp 10.132.0.18:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:38:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:42Z" name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n.18b31a99290b701e namespace: kserve-ci-e2e-test resourceVersion: "15766" uid: 3dcbd61c-0569-4aa7-8092-eb386d11669a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:36:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 2966d957-f88d-4a47-80b9-8657dff04ae4 kind: Event lastTimestamp: "2026-05-26T11:36:59Z" message: 'Created pod: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c-wwx7n' metadata: creationTimestamp: "2026-05-26T11:36:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:36:59Z" name: isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c.18b31a811fcf63e4 namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: e2aa3bb1-725b-43de-9b62-aa2b085b0bb5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:36:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-ea074-predictor namespace: kserve-ci-e2e-test resourceVersion: "14684" uid: 1e74288d-1f4b-4f6b-977d-3e103046a493 kind: Event lastTimestamp: "2026-05-26T11:36:59Z" message: Scaled up replica set isvc-sklearn-graph-raw-ea074-predictor-75d9d67d7c from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:36:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:36:59Z" name: isvc-sklearn-graph-raw-ea074-predictor.18b31a81196b0738 namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: ff5b2b4b-5b14-451b-8bdc-962060eb05d1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:37:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-ea074-predictor namespace: kserve-ci-e2e-test resourceVersion: "14687" uid: 6853aa77-68c3-4385-a27d-e05c02e1f3e0 kind: Event lastTimestamp: "2026-05-26T11:37:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:37:29Z" name: isvc-sklearn-graph-raw-ea074-predictor.18b31a84984b73cf namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: b42d4da8-dc9c-4854-8551-3a7a0bf6c25a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:37:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-ea074-predictor namespace: kserve-ci-e2e-test resourceVersion: "14687" uid: 6853aa77-68c3-4385-a27d-e05c02e1f3e0 kind: Event lastTimestamp: "2026-05-26T11:37:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:37:29Z" name: isvc-sklearn-graph-raw-ea074-predictor.18b31a8498baed8f namespace: kserve-ci-e2e-test resourceVersion: "15021" uid: c053829d-9f15-4b48-ae0f-5e1519e3a2bd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T11:37:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-ea074-predictor namespace: kserve-ci-e2e-test resourceVersion: "14897" uid: 6853aa77-68c3-4385-a27d-e05c02e1f3e0 kind: Event lastTimestamp: "2026-05-26T11:38:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:37:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:38:29Z" name: isvc-sklearn-graph-raw-ea074-predictor.18b31a8b9712e1f7 namespace: kserve-ci-e2e-test resourceVersion: "15508" uid: 67a231fe-c7be-4eec-bec9-4eb290211771 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T11:37:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-ea074-predictor namespace: kserve-ci-e2e-test resourceVersion: "14897" uid: 6853aa77-68c3-4385-a27d-e05c02e1f3e0 kind: Event lastTimestamp: "2026-05-26T11:38:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:37:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:38:29Z" name: isvc-sklearn-graph-raw-ea074-predictor.18b31a8b9785381f namespace: kserve-ci-e2e-test resourceVersion: "15509" uid: da79f30b-c708-4cf2-8391-7d0fe3c8f196 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-26T11:36:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-ea074 namespace: kserve-ci-e2e-test resourceVersion: "14656" uid: 6c7af827-a733-448b-bce0-fa2b12d46087 kind: Event lastTimestamp: "2026-05-26T11:38:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:36:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:38:13Z" name: isvc-sklearn-graph-raw-ea074.18b31a81193449ba namespace: kserve-ci-e2e-test resourceVersion: "15348" uid: f1d86df4-cacf-4aa4-bbdb-83cb3076c44a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:36:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-ea074 namespace: kserve-ci-e2e-test resourceVersion: "14656" uid: 6c7af827-a733-448b-bce0-fa2b12d46087 kind: Event lastTimestamp: "2026-05-26T11:36:59Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-ea074": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-ea074": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:36:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:36:59Z" name: isvc-sklearn-graph-raw-ea074.18b31a8119bed5e8 namespace: kserve-ci-e2e-test resourceVersion: "14692" uid: 1c951e78-3cf6-4682-927b-d2288bd983a3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-ea074 namespace: kserve-ci-e2e-test resourceVersion: "15346" uid: 6c7af827-a733-448b-bce0-fa2b12d46087 kind: Event lastTimestamp: "2026-05-26T11:38:12Z" message: InferenceService [isvc-sklearn-graph-raw-ea074] is Ready metadata: creationTimestamp: "2026-05-26T11:38:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:38:12Z" name: isvc-sklearn-graph-raw-ea074.18b31a9233010406 namespace: kserve-ci-e2e-test resourceVersion: "15347" uid: b2c78572-7ede-45e4-adc1-1096e9f2369e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:38:39.861791Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc namespace: kserve-ci-e2e-test resourceVersion: "15662" uid: f8002f55-cba5-4399-8c87-c8b7a27561b1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc to ip-10-0-132-203.ec2.internal metadata: creationTimestamp: "2026-05-26T11:38:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:38:39Z" name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc.18b31a987631b225 namespace: kserve-ci-e2e-test resourceVersion: "15672" uid: a884df6f-28db-4098-a2fe-dcd001227327 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cfd45b8bd-csfvl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:40Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc namespace: kserve-ci-e2e-test resourceVersion: "15684" uid: f8002f55-cba5-4399-8c87-c8b7a27561b1 kind: Event lastTimestamp: "2026-05-26T11:38:40Z" message: Add eth0 [10.132.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:38:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:38:40Z" name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc.18b31a988f4d403f namespace: kserve-ci-e2e-test resourceVersion: "15722" uid: 2d6177d1-8a45-42f6-bd32-fcc2bc7daa88 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc namespace: kserve-ci-e2e-test resourceVersion: "15667" uid: f8002f55-cba5-4399-8c87-c8b7a27561b1 kind: Event lastTimestamp: "2026-05-26T11:38:40Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:30bffa55f5e3e0e2f6d023c0f84846c206091122d724f5a0e810b4fb957dd389" already present on machine metadata: creationTimestamp: "2026-05-26T11:38:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:40Z" name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc.18b31a989101bdee namespace: kserve-ci-e2e-test resourceVersion: "15724" uid: 899be3c1-e5f0-449a-86ca-3c88ada8f37f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc namespace: kserve-ci-e2e-test resourceVersion: "15667" uid: f8002f55-cba5-4399-8c87-c8b7a27561b1 kind: Event lastTimestamp: "2026-05-26T11:38:40Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T11:38:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:40Z" name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc.18b31a98938cb76e namespace: kserve-ci-e2e-test resourceVersion: "15725" uid: 3a6fbe24-3979-49b7-857e-c9f856f2dc67 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc namespace: kserve-ci-e2e-test resourceVersion: "15667" uid: f8002f55-cba5-4399-8c87-c8b7a27561b1 kind: Event lastTimestamp: "2026-05-26T11:38:40Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T11:38:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:40Z" name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc.18b31a989400672e namespace: kserve-ci-e2e-test resourceVersion: "15726" uid: c72863f9-1779-4725-b7a8-0e9b986258ad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc namespace: kserve-ci-e2e-test resourceVersion: "15667" uid: f8002f55-cba5-4399-8c87-c8b7a27561b1 kind: Event lastTimestamp: "2026-05-26T11:38:44Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T11:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:44Z" name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc.18b31a9976067501 namespace: kserve-ci-e2e-test resourceVersion: "15780" uid: 14644c5c-1d9d-4240-95fa-ed84470db7b2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc namespace: kserve-ci-e2e-test resourceVersion: "15667" uid: f8002f55-cba5-4399-8c87-c8b7a27561b1 kind: Event lastTimestamp: "2026-05-26T11:38:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:44Z" name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc.18b31a9979915669 namespace: kserve-ci-e2e-test resourceVersion: "15788" uid: b9c07b87-580f-43d9-8a15-4b4f8bb1e587 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc namespace: kserve-ci-e2e-test resourceVersion: "15667" uid: f8002f55-cba5-4399-8c87-c8b7a27561b1 kind: Event lastTimestamp: "2026-05-26T11:38:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:44Z" name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc.18b31a997a142d3f namespace: kserve-ci-e2e-test resourceVersion: "15793" uid: bc119e49-accf-4a74-be29-a05d8e5437b6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc namespace: kserve-ci-e2e-test resourceVersion: "15667" uid: f8002f55-cba5-4399-8c87-c8b7a27561b1 kind: Event lastTimestamp: "2026-05-26T11:38:44Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T11:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:44Z" name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc.18b31a997a1e7b09 namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: 9ec69e12-787c-4d2e-bbd4-a9c648bae0b3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc namespace: kserve-ci-e2e-test resourceVersion: "15667" uid: f8002f55-cba5-4399-8c87-c8b7a27561b1 kind: Event lastTimestamp: "2026-05-26T11:38:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:44Z" name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc.18b31a997d14b3ad namespace: kserve-ci-e2e-test resourceVersion: "15798" uid: 16f9fbf0-6acc-4d4e-9e8a-50ac36f598e7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc namespace: kserve-ci-e2e-test resourceVersion: "15667" uid: f8002f55-cba5-4399-8c87-c8b7a27561b1 kind: Event lastTimestamp: "2026-05-26T11:38:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:44Z" name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc.18b31a997d85b310 namespace: kserve-ci-e2e-test resourceVersion: "15801" uid: a1d0b31d-df79-43de-8d8e-840c4054e0eb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-26T11:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc namespace: kserve-ci-e2e-test resourceVersion: "15667" uid: f8002f55-cba5-4399-8c87-c8b7a27561b1 kind: Event lastTimestamp: "2026-05-26T11:40:21Z" message: 'Readiness probe failed: dial tcp 10.132.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:21Z" name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc.18b31a99b209ade9 namespace: kserve-ci-e2e-test resourceVersion: "16666" uid: 7cb698eb-d2bf-49fb-a41c-59f6be58f050 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc namespace: kserve-ci-e2e-test resourceVersion: "15667" uid: f8002f55-cba5-4399-8c87-c8b7a27561b1 kind: Event lastTimestamp: "2026-05-26T11:40:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T11:40:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:20Z" name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc.18b31aafd374dcf8 namespace: kserve-ci-e2e-test resourceVersion: "16635" uid: d5098a80-b3c9-49ed-b5af-65b7ae985999 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc namespace: kserve-ci-e2e-test resourceVersion: "15667" uid: f8002f55-cba5-4399-8c87-c8b7a27561b1 kind: Event lastTimestamp: "2026-05-26T11:40:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:40:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:20Z" name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc.18b31aafd3754c5f namespace: kserve-ci-e2e-test resourceVersion: "16636" uid: edff84cc-6254-4960-b7ea-051a537a7579 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc namespace: kserve-ci-e2e-test resourceVersion: "15667" uid: f8002f55-cba5-4399-8c87-c8b7a27561b1 kind: Event lastTimestamp: "2026-05-26T11:40:21Z" message: 'Readiness probe failed: Get "https://10.132.0.20:8643/healthz": dial tcp 10.132.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:40:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:21Z" name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc.18b31ab00c6934cf namespace: kserve-ci-e2e-test resourceVersion: "16665" uid: e7e43194-e61d-4a33-84de-dc5b9ca0fc65 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7 namespace: kserve-ci-e2e-test resourceVersion: "15633" uid: dfcc10bd-7408-4b3f-9fe6-fafdff7fb2bd kind: Event lastTimestamp: "2026-05-26T11:38:39Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7-tpnqc' metadata: creationTimestamp: "2026-05-26T11:38:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:38:39Z" name: isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7.18b31a9875b5127d namespace: kserve-ci-e2e-test resourceVersion: "15666" uid: c904dd78-c6a1-4f21-91b5-dd4353379964 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-019fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "15628" uid: 9f7cfe56-fa9c-4884-9d0a-eeaad9b49f72 kind: Event lastTimestamp: "2026-05-26T11:38:39Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-019fa-predictor-786c6c85c7 from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:38:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:38:39Z" name: isvc-sklearn-graph-raw-hpa-019fa-predictor.18b31a9870ced80d namespace: kserve-ci-e2e-test resourceVersion: "15638" uid: ab62bd1e-719b-4850-9dba-460ed9d6b13e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:38:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-019fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 2d49e62a-7d60-4a2b-99a4-821f47f86bb0 kind: Event lastTimestamp: "2026-05-26T11:39:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:38:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:39:09Z" name: isvc-sklearn-graph-raw-hpa-019fa-predictor.18b31a9bf0640469 namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: 5eced840-5ddc-4ad7-84de-dfe888d3cee4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:38:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-019fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 2d49e62a-7d60-4a2b-99a4-821f47f86bb0 kind: Event lastTimestamp: "2026-05-26T11:39:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:38:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:39:09Z" name: isvc-sklearn-graph-raw-hpa-019fa-predictor.18b31a9bf0f08f12 namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: a90f83c5-f984-45dd-88a5-efa765a143b4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T11:39:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-019fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: 2d49e62a-7d60-4a2b-99a4-821f47f86bb0 kind: Event lastTimestamp: "2026-05-26T11:40:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:39:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:40:09Z" name: isvc-sklearn-graph-raw-hpa-019fa-predictor.18b31aa2eeff5034 namespace: kserve-ci-e2e-test resourceVersion: "16453" uid: 6f13ef0a-8911-4e2b-a9e6-c604b2dbbea8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T11:39:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-019fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: 2d49e62a-7d60-4a2b-99a4-821f47f86bb0 kind: Event lastTimestamp: "2026-05-26T11:40:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:39:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:40:09Z" name: isvc-sklearn-graph-raw-hpa-019fa-predictor.18b31aa2ef8a9ef4 namespace: kserve-ci-e2e-test resourceVersion: "16455" uid: 274e7f07-b11c-44df-9aa3-04e470e4d9f9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-26T11:38:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-019fa namespace: kserve-ci-e2e-test resourceVersion: "15618" uid: 2c6a19c8-7f79-4166-aff7-8c11dc2c280d kind: Event lastTimestamp: "2026-05-26T11:39:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:38:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:39:51Z" name: isvc-sklearn-graph-raw-hpa-019fa.18b31a9870e4a437 namespace: kserve-ci-e2e-test resourceVersion: "16321" uid: 863199f3-5ea9-4825-a196-da6532e27824 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-019fa namespace: kserve-ci-e2e-test resourceVersion: "15618" uid: 2c6a19c8-7f79-4166-aff7-8c11dc2c280d kind: Event lastTimestamp: "2026-05-26T11:38:39Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-019fa": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-019fa": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:38:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:38:39Z" name: isvc-sklearn-graph-raw-hpa-019fa.18b31a9871b8b1e5 namespace: kserve-ci-e2e-test resourceVersion: "15646" uid: ecb208fa-aea6-4fa1-a3ea-18b5dfb42818 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:39:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-019fa namespace: kserve-ci-e2e-test resourceVersion: "16318" uid: 2c6a19c8-7f79-4166-aff7-8c11dc2c280d kind: Event lastTimestamp: "2026-05-26T11:39:51Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-019fa] is Ready metadata: creationTimestamp: "2026-05-26T11:39:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:39:51Z" name: isvc-sklearn-graph-raw-hpa-019fa.18b31aa915741178 namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 95f6bad4-f25a-4efb-aa7a-41fd033d958b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:42:05.534933Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6 namespace: kserve-ci-e2e-test resourceVersion: "17576" uid: b9da2639-8a5d-4e8d-9db3-0aa42612abfc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6 to ip-10-0-132-203.ec2.internal metadata: creationTimestamp: "2026-05-26T11:42:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:42:05Z" name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6.18b31ac85944c519 namespace: kserve-ci-e2e-test resourceVersion: "17582" uid: a7738e3b-97a5-47ba-b76c-e59000d5ce1f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cfd45b8bd-csfvl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:42:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6 namespace: kserve-ci-e2e-test resourceVersion: "17580" uid: b9da2639-8a5d-4e8d-9db3-0aa42612abfc kind: Event lastTimestamp: "2026-05-26T11:42:05Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-scale-raw-15879-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T11:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:42:05Z" name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6.18b31ac865bf2c52 namespace: kserve-ci-e2e-test resourceVersion: "17605" uid: 824d556d-908f-4558-8c12-26237365d5cb reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:42:06Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6 namespace: kserve-ci-e2e-test resourceVersion: "17586" uid: b9da2639-8a5d-4e8d-9db3-0aa42612abfc kind: Event lastTimestamp: "2026-05-26T11:42:06Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:42:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:42:06Z" name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6.18b31ac8960ec368 namespace: kserve-ci-e2e-test resourceVersion: "17615" uid: 87fd6cf1-a0e4-4f84-8571-75f0ae6eef35 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:42:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6 namespace: kserve-ci-e2e-test resourceVersion: "17580" uid: b9da2639-8a5d-4e8d-9db3-0aa42612abfc kind: Event lastTimestamp: "2026-05-26T11:42:06Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:30bffa55f5e3e0e2f6d023c0f84846c206091122d724f5a0e810b4fb957dd389" already present on machine metadata: creationTimestamp: "2026-05-26T11:42:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:42:06Z" name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6.18b31ac897676033 namespace: kserve-ci-e2e-test resourceVersion: "17617" uid: cd9bf9de-f9d8-460a-9202-b0586a0f1511 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:42:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6 namespace: kserve-ci-e2e-test resourceVersion: "17580" uid: b9da2639-8a5d-4e8d-9db3-0aa42612abfc kind: Event lastTimestamp: "2026-05-26T11:42:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T11:42:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:42:06Z" name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6.18b31ac89a80fe8a namespace: kserve-ci-e2e-test resourceVersion: "17619" uid: 159ac4b4-940c-4bde-b1de-e7e5a2b7429a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:42:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6 namespace: kserve-ci-e2e-test resourceVersion: "17580" uid: b9da2639-8a5d-4e8d-9db3-0aa42612abfc kind: Event lastTimestamp: "2026-05-26T11:42:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T11:42:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:42:06Z" name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6.18b31ac89aef1346 namespace: kserve-ci-e2e-test resourceVersion: "17620" uid: 3569955a-3b0a-4e75-8bcc-2fbfd3513b99 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:42:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6 namespace: kserve-ci-e2e-test resourceVersion: "17580" uid: b9da2639-8a5d-4e8d-9db3-0aa42612abfc kind: Event lastTimestamp: "2026-05-26T11:42:10Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T11:42:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:42:10Z" name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6.18b31ac98f26ed00 namespace: kserve-ci-e2e-test resourceVersion: "17656" uid: 9310ea3e-5785-4975-a17c-888c29eebf5d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:42:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6 namespace: kserve-ci-e2e-test resourceVersion: "17580" uid: b9da2639-8a5d-4e8d-9db3-0aa42612abfc kind: Event lastTimestamp: "2026-05-26T11:42:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:42:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:42:10Z" name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6.18b31ac992288ecf namespace: kserve-ci-e2e-test resourceVersion: "17658" uid: 272959a0-4f9d-4e59-8fa4-d78eff4f3879 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:42:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6 namespace: kserve-ci-e2e-test resourceVersion: "17580" uid: b9da2639-8a5d-4e8d-9db3-0aa42612abfc kind: Event lastTimestamp: "2026-05-26T11:42:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:42:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:42:10Z" name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6.18b31ac992940abb namespace: kserve-ci-e2e-test resourceVersion: "17659" uid: 24424cad-2edc-4392-8c13-9b86455ed648 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:42:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6 namespace: kserve-ci-e2e-test resourceVersion: "17580" uid: b9da2639-8a5d-4e8d-9db3-0aa42612abfc kind: Event lastTimestamp: "2026-05-26T11:42:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T11:42:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:42:10Z" name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6.18b31ac992a00ab5 namespace: kserve-ci-e2e-test resourceVersion: "17660" uid: 9d0acc0b-3500-4d48-9a00-dbcb666af201 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:42:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6 namespace: kserve-ci-e2e-test resourceVersion: "17580" uid: b9da2639-8a5d-4e8d-9db3-0aa42612abfc kind: Event lastTimestamp: "2026-05-26T11:42:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:42:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:42:10Z" name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6.18b31ac995333aa6 namespace: kserve-ci-e2e-test resourceVersion: "17663" uid: 9886ad2d-8b7f-41cb-ac2f-11ab7302a129 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:42:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6 namespace: kserve-ci-e2e-test resourceVersion: "17580" uid: b9da2639-8a5d-4e8d-9db3-0aa42612abfc kind: Event lastTimestamp: "2026-05-26T11:42:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:42:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:42:10Z" name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6.18b31ac9959d3f6b namespace: kserve-ci-e2e-test resourceVersion: "17664" uid: fe75c40b-89a6-4d43-890a-e653c1f699a1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 15 eventTime: null firstTimestamp: "2026-05-26T11:42:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6 namespace: kserve-ci-e2e-test resourceVersion: "17580" uid: b9da2639-8a5d-4e8d-9db3-0aa42612abfc kind: Event lastTimestamp: "2026-05-26T11:44:27Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:42:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:27Z" name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6.18b31aca06dd3691 namespace: kserve-ci-e2e-test resourceVersion: "18719" uid: d7f2b38a-c69a-40f5-a875-ac7bad0c1562 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6 namespace: kserve-ci-e2e-test resourceVersion: "17580" uid: b9da2639-8a5d-4e8d-9db3-0aa42612abfc kind: Event lastTimestamp: "2026-05-26T11:44:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T11:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:25Z" name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6.18b31ae900bc97fe namespace: kserve-ci-e2e-test resourceVersion: "18665" uid: 37d407a3-1f5c-4244-a9cf-84b96fd481f1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6 namespace: kserve-ci-e2e-test resourceVersion: "17580" uid: b9da2639-8a5d-4e8d-9db3-0aa42612abfc kind: Event lastTimestamp: "2026-05-26T11:44:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:25Z" name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6.18b31ae900bd470b namespace: kserve-ci-e2e-test resourceVersion: "18669" uid: 333c04c7-f9ea-4476-a00d-b08aacb22c51 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:44:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6 namespace: kserve-ci-e2e-test resourceVersion: "17580" uid: b9da2639-8a5d-4e8d-9db3-0aa42612abfc kind: Event lastTimestamp: "2026-05-26T11:44:33Z" message: 'Readiness probe failed: Get "https://10.132.0.24:8643/healthz": dial tcp 10.132.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:44:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:33Z" name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6.18b31ae9b134316d namespace: kserve-ci-e2e-test resourceVersion: "18772" uid: efcde5a5-87a9-4ec0-8a76-4f1014d58d67 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:42:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79 namespace: kserve-ci-e2e-test resourceVersion: "17551" uid: 2c89f1f3-fa64-447c-a4e1-6f5144ebd2ec kind: Event lastTimestamp: "2026-05-26T11:42:05Z" message: 'Created pod: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79-stpl6' metadata: creationTimestamp: "2026-05-26T11:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:42:05Z" name: isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79.18b31ac858b111af namespace: kserve-ci-e2e-test resourceVersion: "17579" uid: 53ec3b7a-e90d-460b-9ecd-bafca616f17e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:42:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-15879-predictor namespace: kserve-ci-e2e-test resourceVersion: "17545" uid: 54d511e8-4925-4d5f-8c0c-782a2d19ce30 kind: Event lastTimestamp: "2026-05-26T11:42:05Z" message: Scaled up replica set isvc-sklearn-scale-raw-15879-predictor-6fcdc78f79 from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:42:05Z" name: isvc-sklearn-scale-raw-15879-predictor.18b31ac8552b541e namespace: kserve-ci-e2e-test resourceVersion: "17559" uid: 3865c2eb-4e92-45dd-abf3-224dec5c000d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:42:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-15879-predictor namespace: kserve-ci-e2e-test resourceVersion: "17550" uid: 39bd7ab2-63b4-4355-9fb5-b9afade68df4 kind: Event lastTimestamp: "2026-05-26T11:42:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:42:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:42:50Z" name: isvc-sklearn-scale-raw-15879-predictor.18b31acbd3f785e9 namespace: kserve-ci-e2e-test resourceVersion: "17966" uid: cd0dc5d1-fafe-43ed-a7ba-c2b9b7379fb6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:42:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-15879-predictor namespace: kserve-ci-e2e-test resourceVersion: "17550" uid: 39bd7ab2-63b4-4355-9fb5-b9afade68df4 kind: Event lastTimestamp: "2026-05-26T11:42:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:42:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:42:50Z" name: isvc-sklearn-scale-raw-15879-predictor.18b31acbd460b8d1 namespace: kserve-ci-e2e-test resourceVersion: "17967" uid: 12171501-9cfc-4ba0-946a-4656ad4c0d1d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T11:43:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-15879-predictor namespace: kserve-ci-e2e-test resourceVersion: "17747" uid: 39bd7ab2-63b4-4355-9fb5-b9afade68df4 kind: Event lastTimestamp: "2026-05-26T11:44:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:20Z" name: isvc-sklearn-scale-raw-15879-predictor.18b31ad65187e960 namespace: kserve-ci-e2e-test resourceVersion: "18600" uid: c7cf9e02-e37a-4b34-a636-4e3c1d54d8cd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T11:43:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-15879-predictor namespace: kserve-ci-e2e-test resourceVersion: "17747" uid: 39bd7ab2-63b4-4355-9fb5-b9afade68df4 kind: Event lastTimestamp: "2026-05-26T11:44:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:20Z" name: isvc-sklearn-scale-raw-15879-predictor.18b31ad651f5c11f namespace: kserve-ci-e2e-test resourceVersion: "18601" uid: 1f870528-fb81-431f-a87b-6dad41a996cd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-05-26T11:42:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-15879 namespace: kserve-ci-e2e-test resourceVersion: "17535" uid: 27c337ac-df80-47e5-b63b-344eda3bcc38 kind: Event lastTimestamp: "2026-05-26T11:44:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:44:17Z" name: isvc-sklearn-scale-raw-15879.18b31ac854e751d9 namespace: kserve-ci-e2e-test resourceVersion: "18584" uid: cefebed9-801d-4167-9fe7-9df04bfea056 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:42:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-15879 namespace: kserve-ci-e2e-test resourceVersion: "17535" uid: 27c337ac-df80-47e5-b63b-344eda3bcc38 kind: Event lastTimestamp: "2026-05-26T11:42:05Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-15879": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-15879": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:42:05Z" name: isvc-sklearn-scale-raw-15879.18b31ac855be752d namespace: kserve-ci-e2e-test resourceVersion: "17562" uid: 8ea87cb1-210d-400a-b007-a8ef4b335de4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-15879 namespace: kserve-ci-e2e-test resourceVersion: "18582" uid: 27c337ac-df80-47e5-b63b-344eda3bcc38 kind: Event lastTimestamp: "2026-05-26T11:44:17Z" message: InferenceService [isvc-sklearn-scale-raw-15879] is Ready metadata: creationTimestamp: "2026-05-26T11:44:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:44:17Z" name: isvc-sklearn-scale-raw-15879.18b31ae72aab0d22 namespace: kserve-ci-e2e-test resourceVersion: "18583" uid: ea843acf-f922-42c2-ac6c-5913d6a2e2c6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:36:59.741823Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6 namespace: kserve-ci-e2e-test resourceVersion: "14728" uid: dc0cb65c-ebe3-4a03-a52a-677266309387 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6 to ip-10-0-132-203.ec2.internal metadata: creationTimestamp: "2026-05-26T11:36:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:36:59Z" name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6.18b31a8126944e23 namespace: kserve-ci-e2e-test resourceVersion: "14735" uid: 8f88433e-e762-4093-bbb7-8a838afcd426 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cfd45b8bd-csfvl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:36:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6 namespace: kserve-ci-e2e-test resourceVersion: "14730" uid: dc0cb65c-ebe3-4a03-a52a-677266309387 kind: Event lastTimestamp: "2026-05-26T11:36:59Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-ea074-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T11:36:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:36:59Z" name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6.18b31a812ddece48 namespace: kserve-ci-e2e-test resourceVersion: "14744" uid: 4d42c638-b32e-456f-aa7d-b493bd1f06e0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:37:00Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6 namespace: kserve-ci-e2e-test resourceVersion: "14740" uid: dc0cb65c-ebe3-4a03-a52a-677266309387 kind: Event lastTimestamp: "2026-05-26T11:37:00Z" message: Add eth0 [10.132.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:37:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:37:00Z" name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6.18b31a81635234fc namespace: kserve-ci-e2e-test resourceVersion: "14760" uid: 51ffcc2c-c66b-4266-b5b8-83bee2f5d6e6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:37:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6 namespace: kserve-ci-e2e-test resourceVersion: "14730" uid: dc0cb65c-ebe3-4a03-a52a-677266309387 kind: Event lastTimestamp: "2026-05-26T11:37:00Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:30bffa55f5e3e0e2f6d023c0f84846c206091122d724f5a0e810b4fb957dd389" already present on machine metadata: creationTimestamp: "2026-05-26T11:37:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:37:00Z" name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6.18b31a8164d48e92 namespace: kserve-ci-e2e-test resourceVersion: "14763" uid: f768b465-5dfe-4b21-a0ec-85b833a795e4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:37:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6 namespace: kserve-ci-e2e-test resourceVersion: "14730" uid: dc0cb65c-ebe3-4a03-a52a-677266309387 kind: Event lastTimestamp: "2026-05-26T11:37:00Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T11:37:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:37:00Z" name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6.18b31a8167c2ea82 namespace: kserve-ci-e2e-test resourceVersion: "14764" uid: 273ea6d4-770b-44a8-bf3f-95943c8b2b2d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:37:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6 namespace: kserve-ci-e2e-test resourceVersion: "14730" uid: dc0cb65c-ebe3-4a03-a52a-677266309387 kind: Event lastTimestamp: "2026-05-26T11:37:00Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T11:37:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:37:00Z" name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6.18b31a816822932a namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: 1cba88f2-fb3e-4903-a5bc-5b146e9560a0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:37:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6 namespace: kserve-ci-e2e-test resourceVersion: "14730" uid: dc0cb65c-ebe3-4a03-a52a-677266309387 kind: Event lastTimestamp: "2026-05-26T11:37:04Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-05-26T11:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:37:04Z" name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6.18b31a8256ea8198 namespace: kserve-ci-e2e-test resourceVersion: "14806" uid: 115e3eba-522c-477f-b6df-c298c8cf6279 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:37:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6 namespace: kserve-ci-e2e-test resourceVersion: "14730" uid: dc0cb65c-ebe3-4a03-a52a-677266309387 kind: Event lastTimestamp: "2026-05-26T11:37:24Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 19.531s (19.531s including waiting). Image size: 1306238203 bytes.' metadata: creationTimestamp: "2026-05-26T11:37:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:37:24Z" name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6.18b31a86e316c419 namespace: kserve-ci-e2e-test resourceVersion: "14970" uid: ec1d872b-0f2e-4df4-aaa9-1ed5446e07fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:37:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6 namespace: kserve-ci-e2e-test resourceVersion: "14730" uid: dc0cb65c-ebe3-4a03-a52a-677266309387 kind: Event lastTimestamp: "2026-05-26T11:37:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:37:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:37:24Z" name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6.18b31a86e5ff2c76 namespace: kserve-ci-e2e-test resourceVersion: "14971" uid: e74027e8-505f-4986-8712-bd568f849585 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:37:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6 namespace: kserve-ci-e2e-test resourceVersion: "14730" uid: dc0cb65c-ebe3-4a03-a52a-677266309387 kind: Event lastTimestamp: "2026-05-26T11:37:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:37:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:37:24Z" name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6.18b31a86e6709339 namespace: kserve-ci-e2e-test resourceVersion: "14972" uid: 6fff2565-fa98-44ed-b8b5-6efd46c9d004 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:37:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6 namespace: kserve-ci-e2e-test resourceVersion: "14730" uid: dc0cb65c-ebe3-4a03-a52a-677266309387 kind: Event lastTimestamp: "2026-05-26T11:37:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T11:37:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:37:24Z" name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6.18b31a86e67c4398 namespace: kserve-ci-e2e-test resourceVersion: "14973" uid: acd0604c-a7fe-45b4-a7d2-c90e26c1efef reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:37:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6 namespace: kserve-ci-e2e-test resourceVersion: "14730" uid: dc0cb65c-ebe3-4a03-a52a-677266309387 kind: Event lastTimestamp: "2026-05-26T11:37:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:37:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:37:24Z" name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6.18b31a86e9594b4d namespace: kserve-ci-e2e-test resourceVersion: "14976" uid: 706627b8-8a0d-46df-aee6-2b743940b85c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:37:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6 namespace: kserve-ci-e2e-test resourceVersion: "14730" uid: dc0cb65c-ebe3-4a03-a52a-677266309387 kind: Event lastTimestamp: "2026-05-26T11:37:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:37:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:37:24Z" name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6.18b31a86e9c1ff58 namespace: kserve-ci-e2e-test resourceVersion: "14977" uid: 0e8c9334-c6ee-424b-8b2d-75c4e7ef7af1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-26T11:37:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6 namespace: kserve-ci-e2e-test resourceVersion: "14730" uid: dc0cb65c-ebe3-4a03-a52a-677266309387 kind: Event lastTimestamp: "2026-05-26T11:38:41Z" message: 'Readiness probe failed: dial tcp 10.132.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:37:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:41Z" name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6.18b31a873ed79c13 namespace: kserve-ci-e2e-test resourceVersion: "15756" uid: 4e18f458-7076-4ee6-bf50-4fb75442e330 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6 namespace: kserve-ci-e2e-test resourceVersion: "14730" uid: dc0cb65c-ebe3-4a03-a52a-677266309387 kind: Event lastTimestamp: "2026-05-26T11:38:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T11:38:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:39Z" name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6.18b31a987ded8580 namespace: kserve-ci-e2e-test resourceVersion: "15705" uid: 80943b03-e4b3-4ace-bceb-aa6836e6e015 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6 namespace: kserve-ci-e2e-test resourceVersion: "14730" uid: dc0cb65c-ebe3-4a03-a52a-677266309387 kind: Event lastTimestamp: "2026-05-26T11:38:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:38:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:39Z" name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6.18b31a987dedf875 namespace: kserve-ci-e2e-test resourceVersion: "15707" uid: 6563cf7e-103e-4d19-93af-37c7b15aa383 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6 namespace: kserve-ci-e2e-test resourceVersion: "14730" uid: dc0cb65c-ebe3-4a03-a52a-677266309387 kind: Event lastTimestamp: "2026-05-26T11:38:41Z" message: 'Readiness probe failed: Get "https://10.132.0.19:8643/healthz": dial tcp 10.132.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:38:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:41Z" name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6.18b31a98f0f103ff namespace: kserve-ci-e2e-test resourceVersion: "15755" uid: 82331cc9-5b68-40e7-af53-8cc1d5a48170 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:36:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565 namespace: kserve-ci-e2e-test resourceVersion: "14714" uid: aef65f53-4c0e-4a62-8b26-310952e4c21b kind: Event lastTimestamp: "2026-05-26T11:36:59Z" message: 'Created pod: isvc-xgboost-graph-raw-ea074-predictor-6d59466565-vbtg6' metadata: creationTimestamp: "2026-05-26T11:36:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:36:59Z" name: isvc-xgboost-graph-raw-ea074-predictor-6d59466565.18b31a8125f51471 namespace: kserve-ci-e2e-test resourceVersion: "14729" uid: 731afda7-a207-4455-9ef7-cbb490dc3697 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:36:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-ea074-predictor namespace: kserve-ci-e2e-test resourceVersion: "14711" uid: a5342eba-6f47-46e8-ab22-82e35241bd8f kind: Event lastTimestamp: "2026-05-26T11:36:59Z" message: Scaled up replica set isvc-xgboost-graph-raw-ea074-predictor-6d59466565 from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:36:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:36:59Z" name: isvc-xgboost-graph-raw-ea074-predictor.18b31a8121d33680 namespace: kserve-ci-e2e-test resourceVersion: "14718" uid: def06dc3-64f9-4292-ac88-f74c7df423a9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:37:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-ea074-predictor namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: 2eb97675-7f95-42c2-8465-17a126790c09 kind: Event lastTimestamp: "2026-05-26T11:37:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:37:59Z" name: isvc-xgboost-graph-raw-ea074-predictor.18b31a84a02b6b07 namespace: kserve-ci-e2e-test resourceVersion: "15248" uid: 3cc03dda-a507-428b-96ad-185c5401a79c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:37:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-ea074-predictor namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: 2eb97675-7f95-42c2-8465-17a126790c09 kind: Event lastTimestamp: "2026-05-26T11:37:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:37:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:37:59Z" name: isvc-xgboost-graph-raw-ea074-predictor.18b31a84a0a56b26 namespace: kserve-ci-e2e-test resourceVersion: "15249" uid: 9d02b420-1cf5-485c-a8e2-17881b412691 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:37:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-ea074-predictor namespace: kserve-ci-e2e-test resourceVersion: "14900" uid: 2eb97675-7f95-42c2-8465-17a126790c09 kind: Event lastTimestamp: "2026-05-26T11:38:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:37:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:38:29Z" name: isvc-xgboost-graph-raw-ea074-predictor.18b31a8b9e22b727 namespace: kserve-ci-e2e-test resourceVersion: "15513" uid: 107bc2a5-bf6f-4dbb-b275-43b29c74f438 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:37:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-ea074-predictor namespace: kserve-ci-e2e-test resourceVersion: "14900" uid: 2eb97675-7f95-42c2-8465-17a126790c09 kind: Event lastTimestamp: "2026-05-26T11:38:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:37:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:38:29Z" name: isvc-xgboost-graph-raw-ea074-predictor.18b31a8b9e8b27ae namespace: kserve-ci-e2e-test resourceVersion: "15514" uid: 3f339cd6-e1b6-4b61-a482-bc8fd60e1523 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-26T11:36:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-ea074 namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 348ae90f-9d3d-48cc-8888-57d4299fc705 kind: Event lastTimestamp: "2026-05-26T11:38:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:36:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:38:22Z" name: isvc-xgboost-graph-raw-ea074.18b31a8121850397 namespace: kserve-ci-e2e-test resourceVersion: "15424" uid: 2cd4e132-3fd9-4d5a-bc09-14092a5007be reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-ea074 namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: 348ae90f-9d3d-48cc-8888-57d4299fc705 kind: Event lastTimestamp: "2026-05-26T11:38:22Z" message: InferenceService [isvc-xgboost-graph-raw-ea074] is Ready metadata: creationTimestamp: "2026-05-26T11:38:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:38:22Z" name: isvc-xgboost-graph-raw-ea074.18b31a944f07222c namespace: kserve-ci-e2e-test resourceVersion: "15423" uid: 82533e6a-f194-45ca-b892-e31c9f6f8e47 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:38:40.013901Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5 namespace: kserve-ci-e2e-test resourceVersion: "15708" uid: d44559c0-b91b-4c6a-be3a-c48d06807825 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5 to ip-10-0-132-203.ec2.internal metadata: creationTimestamp: "2026-05-26T11:38:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:38:40Z" name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5.18b31a987f42c448 namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: 0d61d5dc-28ff-4007-ba8c-9e75d95d71f2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cfd45b8bd-csfvl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:40Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5 namespace: kserve-ci-e2e-test resourceVersion: "15716" uid: d44559c0-b91b-4c6a-be3a-c48d06807825 kind: Event lastTimestamp: "2026-05-26T11:38:40Z" message: Add eth0 [10.132.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:38:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:38:40Z" name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5.18b31a9899079e9e namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: dd787864-2ded-4385-a8f1-bd1a7183966a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5 namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: d44559c0-b91b-4c6a-be3a-c48d06807825 kind: Event lastTimestamp: "2026-05-26T11:38:40Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:30bffa55f5e3e0e2f6d023c0f84846c206091122d724f5a0e810b4fb957dd389" already present on machine metadata: creationTimestamp: "2026-05-26T11:38:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:40Z" name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5.18b31a989a6b03c3 namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: 0405bad7-acca-4aeb-9add-a4346386f1fc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5 namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: d44559c0-b91b-4c6a-be3a-c48d06807825 kind: Event lastTimestamp: "2026-05-26T11:38:40Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T11:38:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:40Z" name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5.18b31a989cea27b1 namespace: kserve-ci-e2e-test resourceVersion: "15732" uid: d679e997-cdf5-4de4-ab01-590b83766135 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5 namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: d44559c0-b91b-4c6a-be3a-c48d06807825 kind: Event lastTimestamp: "2026-05-26T11:38:40Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T11:38:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:40Z" name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5.18b31a989d4a23dd namespace: kserve-ci-e2e-test resourceVersion: "15733" uid: 2231f5f3-e1ae-4a5e-bb61-ee6c92cf6e70 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5 namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: d44559c0-b91b-4c6a-be3a-c48d06807825 kind: Event lastTimestamp: "2026-05-26T11:38:44Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-05-26T11:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:44Z" name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5.18b31a9975d767fc namespace: kserve-ci-e2e-test resourceVersion: "15779" uid: a2e5c601-42c7-483a-80ad-ec8f62ff8457 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5 namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: d44559c0-b91b-4c6a-be3a-c48d06807825 kind: Event lastTimestamp: "2026-05-26T11:38:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:44Z" name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5.18b31a9979781149 namespace: kserve-ci-e2e-test resourceVersion: "15786" uid: 7d08cc38-ac04-40ec-ab5b-a7eecd3fe852 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5 namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: d44559c0-b91b-4c6a-be3a-c48d06807825 kind: Event lastTimestamp: "2026-05-26T11:38:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:44Z" name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5.18b31a9979fc97d6 namespace: kserve-ci-e2e-test resourceVersion: "15790" uid: aaccb559-fa3b-4094-83e5-f7504b6710cf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5 namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: d44559c0-b91b-4c6a-be3a-c48d06807825 kind: Event lastTimestamp: "2026-05-26T11:38:44Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T11:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:44Z" name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5.18b31a997a089709 namespace: kserve-ci-e2e-test resourceVersion: "15791" uid: 7aa257a2-f90d-4bb3-8dfb-af1a21652431 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5 namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: d44559c0-b91b-4c6a-be3a-c48d06807825 kind: Event lastTimestamp: "2026-05-26T11:38:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:44Z" name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5.18b31a997cf7f24d namespace: kserve-ci-e2e-test resourceVersion: "15797" uid: 076b9a67-ed21-4fa0-89ea-59c1fcf8bb92 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5 namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: d44559c0-b91b-4c6a-be3a-c48d06807825 kind: Event lastTimestamp: "2026-05-26T11:38:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:44Z" name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5.18b31a997d6a7ce9 namespace: kserve-ci-e2e-test resourceVersion: "15800" uid: f033d162-1664-486d-bf96-2670e9cec173 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-26T11:38:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5 namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: d44559c0-b91b-4c6a-be3a-c48d06807825 kind: Event lastTimestamp: "2026-05-26T11:40:22Z" message: 'Readiness probe failed: dial tcp 10.132.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:22Z" name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5.18b31a99ee0207e6 namespace: kserve-ci-e2e-test resourceVersion: "16676" uid: 696c4230-5e82-4443-a731-bfd8cde32e46 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5 namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: d44559c0-b91b-4c6a-be3a-c48d06807825 kind: Event lastTimestamp: "2026-05-26T11:40:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T11:40:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:20Z" name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5.18b31aafd06a7047 namespace: kserve-ci-e2e-test resourceVersion: "16612" uid: 25606e70-2e50-4f89-b634-8ce5f2a64c9e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5 namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: d44559c0-b91b-4c6a-be3a-c48d06807825 kind: Event lastTimestamp: "2026-05-26T11:40:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:40:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:20Z" name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5.18b31aafd06acab1 namespace: kserve-ci-e2e-test resourceVersion: "16617" uid: 8047216d-631a-41b7-bfc0-b77724edc61b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5 namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: d44559c0-b91b-4c6a-be3a-c48d06807825 kind: Event lastTimestamp: "2026-05-26T11:40:22Z" message: 'Readiness probe failed: Get "https://10.132.0.21:8643/healthz": dial tcp 10.132.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:22Z" name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5.18b31ab0483072ca namespace: kserve-ci-e2e-test resourceVersion: "16675" uid: 47f8ac4f-2a74-44ee-8657-70e06a3029d0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c namespace: kserve-ci-e2e-test resourceVersion: "15671" uid: 2ec18f16-42cc-4ac3-993b-cd78d5059780 kind: Event lastTimestamp: "2026-05-26T11:38:40Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c-nh2d5' metadata: creationTimestamp: "2026-05-26T11:38:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:38:40Z" name: isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c.18b31a987ed91ae7 namespace: kserve-ci-e2e-test resourceVersion: "15709" uid: 4841c68d-70a1-49d0-a9aa-4defca7a04ea reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-019fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 8bf0b635-89eb-4e44-8b5e-4ee66c296ca0 kind: Event lastTimestamp: "2026-05-26T11:38:39Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-019fa-predictor-855d9c6b9c from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:38:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:38:39Z" name: isvc-xgboost-graph-raw-hpa-019fa-predictor.18b31a9876a90c5a namespace: kserve-ci-e2e-test resourceVersion: "15675" uid: ed389637-0522-498b-8612-2f82ed1aa36c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:38:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-019fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "15669" uid: 5c94187a-5ed1-4124-9fec-71c4ce2ce368 kind: Event lastTimestamp: "2026-05-26T11:39:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:38:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:39:09Z" name: isvc-xgboost-graph-raw-hpa-019fa-predictor.18b31a9bf6249c75 namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: 83bacfc9-68ef-456a-9f42-592b97b6e764 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:38:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-019fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "15669" uid: 5c94187a-5ed1-4124-9fec-71c4ce2ce368 kind: Event lastTimestamp: "2026-05-26T11:39:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:38:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:39:09Z" name: isvc-xgboost-graph-raw-hpa-019fa-predictor.18b31a9bf6f8df1f namespace: kserve-ci-e2e-test resourceVersion: "16005" uid: b7eb67c5-fd03-4603-aab6-770c7a5acc0b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:39:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-019fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "15894" uid: 5c94187a-5ed1-4124-9fec-71c4ce2ce368 kind: Event lastTimestamp: "2026-05-26T11:39:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:39:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:39:54Z" name: isvc-xgboost-graph-raw-hpa-019fa-predictor.18b31aa2f486dfa6 namespace: kserve-ci-e2e-test resourceVersion: "16348" uid: 8453c692-3170-416f-970e-3005d6554bc7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:39:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-019fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "15894" uid: 5c94187a-5ed1-4124-9fec-71c4ce2ce368 kind: Event lastTimestamp: "2026-05-26T11:39:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:39:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:39:54Z" name: isvc-xgboost-graph-raw-hpa-019fa-predictor.18b31aa2f4edcd3c namespace: kserve-ci-e2e-test resourceVersion: "16349" uid: ac88d7e4-8583-41ac-a8e2-e545dfbb73df reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-26T11:38:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-019fa namespace: kserve-ci-e2e-test resourceVersion: "15653" uid: 5173c657-2218-4f74-bf4c-21f1ca02d913 kind: Event lastTimestamp: "2026-05-26T11:39:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:38:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:39:42Z" name: isvc-xgboost-graph-raw-hpa-019fa.18b31a9876a689a0 namespace: kserve-ci-e2e-test resourceVersion: "16253" uid: 84d419c7-e583-48ce-afba-e60aaa8d3a93 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-019fa namespace: kserve-ci-e2e-test resourceVersion: "15653" uid: 5173c657-2218-4f74-bf4c-21f1ca02d913 kind: Event lastTimestamp: "2026-05-26T11:38:39Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-hpa-019fa": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-hpa-019fa": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:38:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:38:39Z" name: isvc-xgboost-graph-raw-hpa-019fa.18b31a987717f57e namespace: kserve-ci-e2e-test resourceVersion: "15680" uid: a96dd41c-02d1-4ece-80ff-dee81e94e3c6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:39:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-019fa namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: 5173c657-2218-4f74-bf4c-21f1ca02d913 kind: Event lastTimestamp: "2026-05-26T11:39:42Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-019fa] is Ready metadata: creationTimestamp: "2026-05-26T11:39:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:39:42Z" name: isvc-xgboost-graph-raw-hpa-019fa.18b31aa6fccdb5bf namespace: kserve-ci-e2e-test resourceVersion: "16251" uid: 989fa5e2-539a-4131-b87d-fcfe68836eb2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:40:20.245531Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-a937a-predictor-75bcd59b59-2xqn4 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: ead048f4-f464-4e54-909c-46352edeb48c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-a937a-predictor-75bcd59b59-2xqn4 to ip-10-0-132-203.ec2.internal metadata: creationTimestamp: "2026-05-26T11:40:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:40:20Z" name: message-dumper-raw-a937a-predictor-75bcd59b59-2xqn4.18b31aafd58817c4 namespace: kserve-ci-e2e-test resourceVersion: "16645" uid: 729d2c06-be76-4bc7-89e3-89b613cad961 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cfd45b8bd-csfvl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:20Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-a937a-predictor-75bcd59b59-2xqn4 namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: ead048f4-f464-4e54-909c-46352edeb48c kind: Event lastTimestamp: "2026-05-26T11:40:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "message-dumper-raw-a937a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T11:40:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:20Z" name: message-dumper-raw-a937a-predictor-75bcd59b59-2xqn4.18b31aafe0377c6e namespace: kserve-ci-e2e-test resourceVersion: "16654" uid: 248f3c01-d180-4f58-a996-e761a79d40e5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:21Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-a937a-predictor-75bcd59b59-2xqn4 namespace: kserve-ci-e2e-test resourceVersion: "16649" uid: ead048f4-f464-4e54-909c-46352edeb48c kind: Event lastTimestamp: "2026-05-26T11:40:21Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:40:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:40:21Z" name: message-dumper-raw-a937a-predictor-75bcd59b59-2xqn4.18b31ab0124f91e2 namespace: kserve-ci-e2e-test resourceVersion: "16667" uid: 2dcb1f51-1e77-49c6-a68d-1a13cd495659 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-a937a-predictor-75bcd59b59-2xqn4 namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: ead048f4-f464-4e54-909c-46352edeb48c kind: Event lastTimestamp: "2026-05-26T11:40:21Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-05-26T11:40:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:21Z" name: message-dumper-raw-a937a-predictor-75bcd59b59-2xqn4.18b31ab01387710d namespace: kserve-ci-e2e-test resourceVersion: "16669" uid: 460795b4-d901-442d-8036-91105b9eb2d0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-a937a-predictor-75bcd59b59-2xqn4 namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: ead048f4-f464-4e54-909c-46352edeb48c kind: Event lastTimestamp: "2026-05-26T11:40:22Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.431s (1.431s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-05-26T11:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:22Z" name: message-dumper-raw-a937a-predictor-75bcd59b59-2xqn4.18b31ab068db6062 namespace: kserve-ci-e2e-test resourceVersion: "16681" uid: 3c7bc111-bce7-4985-953e-062f4c2f8bb7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-a937a-predictor-75bcd59b59-2xqn4 namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: ead048f4-f464-4e54-909c-46352edeb48c kind: Event lastTimestamp: "2026-05-26T11:40:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:22Z" name: message-dumper-raw-a937a-predictor-75bcd59b59-2xqn4.18b31ab06bc9bc4c namespace: kserve-ci-e2e-test resourceVersion: "16682" uid: 27c659ed-b9d3-4d57-a7df-aab2126959c8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-a937a-predictor-75bcd59b59-2xqn4 namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: ead048f4-f464-4e54-909c-46352edeb48c kind: Event lastTimestamp: "2026-05-26T11:40:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:22Z" name: message-dumper-raw-a937a-predictor-75bcd59b59-2xqn4.18b31ab06c3aa142 namespace: kserve-ci-e2e-test resourceVersion: "16683" uid: d8cdf552-3e75-421e-a4fc-6b94f6bc3512 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-a937a-predictor-75bcd59b59-2xqn4 namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: ead048f4-f464-4e54-909c-46352edeb48c kind: Event lastTimestamp: "2026-05-26T11:40:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T11:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:22Z" name: message-dumper-raw-a937a-predictor-75bcd59b59-2xqn4.18b31ab06c4711ab namespace: kserve-ci-e2e-test resourceVersion: "16684" uid: 007f9040-f41a-4e4a-ae60-cdb475a23be4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-a937a-predictor-75bcd59b59-2xqn4 namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: ead048f4-f464-4e54-909c-46352edeb48c kind: Event lastTimestamp: "2026-05-26T11:40:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:22Z" name: message-dumper-raw-a937a-predictor-75bcd59b59-2xqn4.18b31ab06ed2b734 namespace: kserve-ci-e2e-test resourceVersion: "16685" uid: 5f656c64-df2b-41f5-b9c7-d9564fa29ec8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-a937a-predictor-75bcd59b59-2xqn4 namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: ead048f4-f464-4e54-909c-46352edeb48c kind: Event lastTimestamp: "2026-05-26T11:40:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:22Z" name: message-dumper-raw-a937a-predictor-75bcd59b59-2xqn4.18b31ab06f3eac18 namespace: kserve-ci-e2e-test resourceVersion: "16686" uid: 516c3a97-6075-46a1-9915-732328081213 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:42:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-a937a-predictor-75bcd59b59-2xqn4 namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: ead048f4-f464-4e54-909c-46352edeb48c kind: Event lastTimestamp: "2026-05-26T11:42:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T11:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:42:05Z" name: message-dumper-raw-a937a-predictor-75bcd59b59-2xqn4.18b31ac85deb0334 namespace: kserve-ci-e2e-test resourceVersion: "17602" uid: a637a11d-0088-4202-b45b-1d65b9e65cd2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:42:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-a937a-predictor-75bcd59b59-2xqn4 namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: ead048f4-f464-4e54-909c-46352edeb48c kind: Event lastTimestamp: "2026-05-26T11:42:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:42:05Z" name: message-dumper-raw-a937a-predictor-75bcd59b59-2xqn4.18b31ac85deb629b namespace: kserve-ci-e2e-test resourceVersion: "17603" uid: 01f9f07e-6d52-4722-89ab-d3dbd495666d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-a937a-predictor-75bcd59b59 namespace: kserve-ci-e2e-test resourceVersion: "16618" uid: 8bd9b9bc-9a54-4de8-bd69-68467710c444 kind: Event lastTimestamp: "2026-05-26T11:40:20Z" message: 'Created pod: message-dumper-raw-a937a-predictor-75bcd59b59-2xqn4' metadata: creationTimestamp: "2026-05-26T11:40:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:40:20Z" name: message-dumper-raw-a937a-predictor-75bcd59b59.18b31aafd50b1a2e namespace: kserve-ci-e2e-test resourceVersion: "16640" uid: 358c08fd-6f6f-479a-b6a1-6df223f7b04b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-a937a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 9e0e55d8-9737-4bf3-93c3-5ae57c46bd25 kind: Event lastTimestamp: "2026-05-26T11:40:20Z" message: Scaled up replica set message-dumper-raw-a937a-predictor-75bcd59b59 from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:40:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:40:20Z" name: message-dumper-raw-a937a-predictor.18b31aafd16d33c0 namespace: kserve-ci-e2e-test resourceVersion: "16623" uid: 5c5276aa-c170-4844-bcf2-fdf35382095f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:40:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-a937a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16616" uid: 0c2e1525-4778-454e-9fe1-5b30a88018b9 kind: Event lastTimestamp: "2026-05-26T11:41:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:41:05Z" name: message-dumper-raw-a937a-predictor.18b31ab3500e85b3 namespace: kserve-ci-e2e-test resourceVersion: "17096" uid: 49407b02-ae2d-4208-901e-45cf39932f24 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:40:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-a937a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16616" uid: 0c2e1525-4778-454e-9fe1-5b30a88018b9 kind: Event lastTimestamp: "2026-05-26T11:41:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:41:05Z" name: message-dumper-raw-a937a-predictor.18b31ab350684289 namespace: kserve-ci-e2e-test resourceVersion: "17097" uid: 4c629a0c-a009-43fa-8124-5efda931c1a9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-a937a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16799" uid: 0c2e1525-4778-454e-9fe1-5b30a88018b9 kind: Event lastTimestamp: "2026-05-26T11:40:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:40:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:40:50Z" name: message-dumper-raw-a937a-predictor.18b31ab6cf713cec namespace: kserve-ci-e2e-test resourceVersion: "16977" uid: 27f842eb-31ed-4ac9-a312-2e758e0cf14f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-a937a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16799" uid: 0c2e1525-4778-454e-9fe1-5b30a88018b9 kind: Event lastTimestamp: "2026-05-26T11:40:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:40:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:40:50Z" name: message-dumper-raw-a937a-predictor.18b31ab6cfd2d6d3 namespace: kserve-ci-e2e-test resourceVersion: "16980" uid: b101d065-7c38-4e80-b284-14a680242eac reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-26T11:40:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-a937a namespace: kserve-ci-e2e-test resourceVersion: "16585" uid: 871da04c-65ce-4d51-93c1-a24bb4545e10 kind: Event lastTimestamp: "2026-05-26T11:40:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:40:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:40:31Z" name: message-dumper-raw-a937a.18b31aafd1549ea8 namespace: kserve-ci-e2e-test resourceVersion: "16772" uid: 5c13e596-d39f-4347-9497-73941e8d5fb2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-a937a namespace: kserve-ci-e2e-test resourceVersion: "16585" uid: 871da04c-65ce-4d51-93c1-a24bb4545e10 kind: Event lastTimestamp: "2026-05-26T11:40:20Z" message: 'Failed to update status for InferenceService "message-dumper-raw-a937a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-a937a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:40:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:40:20Z" name: message-dumper-raw-a937a.18b31aafd20a1c64 namespace: kserve-ci-e2e-test resourceVersion: "16628" uid: 76da31bc-d35d-449e-af63-ad80d07d4403 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-a937a namespace: kserve-ci-e2e-test resourceVersion: "16769" uid: 871da04c-65ce-4d51-93c1-a24bb4545e10 kind: Event lastTimestamp: "2026-05-26T11:40:31Z" message: InferenceService [message-dumper-raw-a937a] is Ready metadata: creationTimestamp: "2026-05-26T11:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:40:31Z" name: message-dumper-raw-a937a.18b31ab275f99cd0 namespace: kserve-ci-e2e-test resourceVersion: "16771" uid: d70f0bf2-f89b-4c49-9580-95878cb86fa8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:38:29.591029Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-ea074-5dbf4895df-jgrqc namespace: kserve-ci-e2e-test resourceVersion: "15494" uid: c7810e0a-836d-42d6-a847-a6bebe271171 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-ea074-5dbf4895df-jgrqc to ip-10-0-130-251.ec2.internal metadata: creationTimestamp: "2026-05-26T11:38:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:38:29Z" name: model-chainer-raw-ea074-5dbf4895df-jgrqc.18b31a961202557e namespace: kserve-ci-e2e-test resourceVersion: "15499" uid: 1b79e1bd-9b5e-4b7b-8b66-7fe6475abe0f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cfd45b8bd-csfvl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:30Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-ea074-5dbf4895df-jgrqc namespace: kserve-ci-e2e-test resourceVersion: "15503" uid: c7810e0a-836d-42d6-a847-a6bebe271171 kind: Event lastTimestamp: "2026-05-26T11:38:30Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:38:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:38:30Z" name: model-chainer-raw-ea074-5dbf4895df-jgrqc.18b31a962b78a4eb namespace: kserve-ci-e2e-test resourceVersion: "15517" uid: 6e46d0f7-dd49-4351-bd0f-d071929ca42d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-ea074} kind: Pod name: model-chainer-raw-ea074-5dbf4895df-jgrqc namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: c7810e0a-836d-42d6-a847-a6bebe271171 kind: Event lastTimestamp: "2026-05-26T11:38:30Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" metadata: creationTimestamp: "2026-05-26T11:38:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:30Z" name: model-chainer-raw-ea074-5dbf4895df-jgrqc.18b31a962cab481b namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: 8adcf97b-948c-4433-997b-3709455b80b5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-251.ec2.internal source: component: kubelet host: ip-10-0-130-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-ea074} kind: Pod name: model-chainer-raw-ea074-5dbf4895df-jgrqc namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: c7810e0a-836d-42d6-a847-a6bebe271171 kind: Event lastTimestamp: "2026-05-26T11:38:31Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" in 1.961s (1.961s including waiting). Image size: 217997404 bytes.' metadata: creationTimestamp: "2026-05-26T11:38:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:32Z" name: model-chainer-raw-ea074-5dbf4895df-jgrqc.18b31a96a1927a1d namespace: kserve-ci-e2e-test resourceVersion: "15533" uid: d9e264bc-a222-4f65-a8d1-4523859480fe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-251.ec2.internal source: component: kubelet host: ip-10-0-130-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-ea074} kind: Pod name: model-chainer-raw-ea074-5dbf4895df-jgrqc namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: c7810e0a-836d-42d6-a847-a6bebe271171 kind: Event lastTimestamp: "2026-05-26T11:38:32Z" message: 'Created container: model-chainer-raw-ea074' metadata: creationTimestamp: "2026-05-26T11:38:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:32Z" name: model-chainer-raw-ea074-5dbf4895df-jgrqc.18b31a96a4e74127 namespace: kserve-ci-e2e-test resourceVersion: "15534" uid: 87c9d4c7-f72b-4512-9620-f324c58d94b0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-251.ec2.internal source: component: kubelet host: ip-10-0-130-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-ea074} kind: Pod name: model-chainer-raw-ea074-5dbf4895df-jgrqc namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: c7810e0a-836d-42d6-a847-a6bebe271171 kind: Event lastTimestamp: "2026-05-26T11:38:32Z" message: Started container model-chainer-raw-ea074 metadata: creationTimestamp: "2026-05-26T11:38:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:32Z" name: model-chainer-raw-ea074-5dbf4895df-jgrqc.18b31a96a56034c0 namespace: kserve-ci-e2e-test resourceVersion: "15535" uid: 5160244d-81b5-42d6-a6aa-c04bcdb9f3d2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-251.ec2.internal source: component: kubelet host: ip-10-0-130-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-ea074} kind: Pod name: model-chainer-raw-ea074-5dbf4895df-jgrqc namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: c7810e0a-836d-42d6-a847-a6bebe271171 kind: Event lastTimestamp: "2026-05-26T11:38:39Z" message: Stopping container model-chainer-raw-ea074 metadata: creationTimestamp: "2026-05-26T11:38:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:38:39Z" name: model-chainer-raw-ea074-5dbf4895df-jgrqc.18b31a986957af61 namespace: kserve-ci-e2e-test resourceVersion: "15615" uid: 600acf46-d678-4496-aca0-9ac12d9340f7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-251.ec2.internal source: component: kubelet host: ip-10-0-130-251.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T11:38:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-ea074} kind: Pod name: model-chainer-raw-ea074-5dbf4895df-jgrqc namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: c7810e0a-836d-42d6-a847-a6bebe271171 kind: Event lastTimestamp: "2026-05-26T11:39:08Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T11:38:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:39:08Z" name: model-chainer-raw-ea074-5dbf4895df-jgrqc.18b31a9957156a43 namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: 8279b098-9bd7-4717-bc8a-5185cef8f2a2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-251.ec2.internal source: component: kubelet host: ip-10-0-130-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-ea074-5dbf4895df namespace: kserve-ci-e2e-test resourceVersion: "15491" uid: 3a63f244-a81f-422c-b172-f47a47b6f83c kind: Event lastTimestamp: "2026-05-26T11:38:29Z" message: 'Created pod: model-chainer-raw-ea074-5dbf4895df-jgrqc' metadata: creationTimestamp: "2026-05-26T11:38:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:38:29Z" name: model-chainer-raw-ea074-5dbf4895df.18b31a96114f05fa namespace: kserve-ci-e2e-test resourceVersion: "15495" uid: 0e46dde0-a1ed-4c43-a1c5-6152783c03ef reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-ea074 namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: 9f336f2d-2f7a-4650-8e59-ac28a8a930c3 kind: Event lastTimestamp: "2026-05-26T11:38:29Z" message: Scaled up replica set model-chainer-raw-ea074-5dbf4895df from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:38:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:38:29Z" name: model-chainer-raw-ea074.18b31a96108fba72 namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: 3cc916f3-9fcd-41cc-a276-02eabf588119 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:38:29Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-ea074 namespace: kserve-ci-e2e-test resourceVersion: "15505" uid: 87d9306f-b30f-4527-b6ca-71d25a4c4694 kind: Event lastTimestamp: "2026-05-26T11:38:29Z" message: InferenceGraph [model-chainer-raw-ea074] is Ready metadata: creationTimestamp: "2026-05-26T11:38:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:38:29Z" name: model-chainer-raw-ea074.18b31a9614829923 namespace: kserve-ci-e2e-test resourceVersion: "15507" uid: 0eb69a79-c948-402b-8077-27f5c2748508 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:40:09.969173Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-019fa-d557f9465-cwxx6 namespace: kserve-ci-e2e-test resourceVersion: "16465" uid: 816b3fc7-b3a4-4efa-9efc-d7daeffd976f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-019fa-d557f9465-cwxx6 to ip-10-0-130-251.ec2.internal metadata: creationTimestamp: "2026-05-26T11:40:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:40:09Z" name: model-chainer-raw-hpa-019fa-d557f9465-cwxx6.18b31aad71033bef namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 01c4ffa2-5f4b-4946-bb9c-3c44b43a4c75 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cfd45b8bd-csfvl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:10Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-019fa-d557f9465-cwxx6 namespace: kserve-ci-e2e-test resourceVersion: "16475" uid: 816b3fc7-b3a4-4efa-9efc-d7daeffd976f kind: Event lastTimestamp: "2026-05-26T11:40:10Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:40:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:40:10Z" name: model-chainer-raw-hpa-019fa-d557f9465-cwxx6.18b31aad8a55f7b7 namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 0e7f4ef5-cc07-4d32-831c-d904821ca6fe reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-019fa} kind: Pod name: model-chainer-raw-hpa-019fa-d557f9465-cwxx6 namespace: kserve-ci-e2e-test resourceVersion: "16467" uid: 816b3fc7-b3a4-4efa-9efc-d7daeffd976f kind: Event lastTimestamp: "2026-05-26T11:40:10Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" already present on machine metadata: creationTimestamp: "2026-05-26T11:40:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:10Z" name: model-chainer-raw-hpa-019fa-d557f9465-cwxx6.18b31aad8b929b40 namespace: kserve-ci-e2e-test resourceVersion: "16488" uid: 4b9fe3a9-5a17-4a11-ac6f-050b34fdc474 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-251.ec2.internal source: component: kubelet host: ip-10-0-130-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-019fa} kind: Pod name: model-chainer-raw-hpa-019fa-d557f9465-cwxx6 namespace: kserve-ci-e2e-test resourceVersion: "16467" uid: 816b3fc7-b3a4-4efa-9efc-d7daeffd976f kind: Event lastTimestamp: "2026-05-26T11:40:10Z" message: 'Created container: model-chainer-raw-hpa-019fa' metadata: creationTimestamp: "2026-05-26T11:40:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:10Z" name: model-chainer-raw-hpa-019fa-d557f9465-cwxx6.18b31aad8e6e9bbf namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 1b777ef0-4c81-44c8-976f-ef51ab37d669 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-251.ec2.internal source: component: kubelet host: ip-10-0-130-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-019fa} kind: Pod name: model-chainer-raw-hpa-019fa-d557f9465-cwxx6 namespace: kserve-ci-e2e-test resourceVersion: "16467" uid: 816b3fc7-b3a4-4efa-9efc-d7daeffd976f kind: Event lastTimestamp: "2026-05-26T11:40:10Z" message: Started container model-chainer-raw-hpa-019fa metadata: creationTimestamp: "2026-05-26T11:40:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:10Z" name: model-chainer-raw-hpa-019fa-d557f9465-cwxx6.18b31aad8edb8b75 namespace: kserve-ci-e2e-test resourceVersion: "16490" uid: 257a9add-4624-4720-9326-dbb5ef400f00 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-251.ec2.internal source: component: kubelet host: ip-10-0-130-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-019fa} kind: Pod name: model-chainer-raw-hpa-019fa-d557f9465-cwxx6 namespace: kserve-ci-e2e-test resourceVersion: "16467" uid: 816b3fc7-b3a4-4efa-9efc-d7daeffd976f kind: Event lastTimestamp: "2026-05-26T11:40:20Z" message: Stopping container model-chainer-raw-hpa-019fa metadata: creationTimestamp: "2026-05-26T11:40:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:20Z" name: model-chainer-raw-hpa-019fa-d557f9465-cwxx6.18b31aafc7e52874 namespace: kserve-ci-e2e-test resourceVersion: "16582" uid: f892f730-e49d-4dfb-a9f5-99ba33c89270 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-251.ec2.internal source: component: kubelet host: ip-10-0-130-251.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T11:40:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-019fa} kind: Pod name: model-chainer-raw-hpa-019fa-d557f9465-cwxx6 namespace: kserve-ci-e2e-test resourceVersion: "16467" uid: 816b3fc7-b3a4-4efa-9efc-d7daeffd976f kind: Event lastTimestamp: "2026-05-26T11:40:46Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T11:40:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:40:46Z" name: model-chainer-raw-hpa-019fa-d557f9465-cwxx6.18b31ab03c4d1118 namespace: kserve-ci-e2e-test resourceVersion: "16950" uid: 1a03c41e-7b84-43c2-8314-272985f093ef reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-251.ec2.internal source: component: kubelet host: ip-10-0-130-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-019fa-d557f9465 namespace: kserve-ci-e2e-test resourceVersion: "16462" uid: 627ba9f7-3ca8-4ed7-b322-8d4da70491e6 kind: Event lastTimestamp: "2026-05-26T11:40:09Z" message: 'Created pod: model-chainer-raw-hpa-019fa-d557f9465-cwxx6' metadata: creationTimestamp: "2026-05-26T11:40:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:40:09Z" name: model-chainer-raw-hpa-019fa-d557f9465.18b31aad70404bfb namespace: kserve-ci-e2e-test resourceVersion: "16466" uid: f172c982-4ccf-4537-b190-e4aeaa594710 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-019fa namespace: kserve-ci-e2e-test resourceVersion: "16460" uid: b975f867-17d0-481d-88a7-c205394c6eee kind: Event lastTimestamp: "2026-05-26T11:40:09Z" message: Scaled up replica set model-chainer-raw-hpa-019fa-d557f9465 from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:40:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:40:09Z" name: model-chainer-raw-hpa-019fa.18b31aad6f598e24 namespace: kserve-ci-e2e-test resourceVersion: "16463" uid: e7c6a729-1eca-48b1-8b04-a873c17fa256 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:40:10Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-019fa namespace: kserve-ci-e2e-test resourceVersion: "16477" uid: 732348c9-9dc9-4ff3-8388-209c1c72b489 kind: Event lastTimestamp: "2026-05-26T11:40:10Z" message: InferenceGraph [model-chainer-raw-hpa-019fa] is Ready metadata: creationTimestamp: "2026-05-26T11:40:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:40:10Z" name: model-chainer-raw-hpa-019fa.18b31aad750a1f5b namespace: kserve-ci-e2e-test resourceVersion: "16479" uid: a7dd8c2b-8e24-40a9-9faf-7d767727c203 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:46:10.351941Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4 namespace: kserve-ci-e2e-test resourceVersion: "19691" uid: 4eb67ebf-1d83-4854-8077-8ed885de3d89 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4 to ip-10-0-132-203.ec2.internal metadata: creationTimestamp: "2026-05-26T11:46:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:46:10Z" name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4.18b31b01597fed0f namespace: kserve-ci-e2e-test resourceVersion: "19698" uid: 89abf630-51a4-42c7-85de-7868e6f8435e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cfd45b8bd-csfvl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:10Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4 namespace: kserve-ci-e2e-test resourceVersion: "19704" uid: 4eb67ebf-1d83-4854-8077-8ed885de3d89 kind: Event lastTimestamp: "2026-05-26T11:46:10Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:46:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:46:10Z" name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4.18b31b017268996f namespace: kserve-ci-e2e-test resourceVersion: "19715" uid: c5a27b73-2fd1-4be9-acde-6944da8dc95f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4 namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 4eb67ebf-1d83-4854-8077-8ed885de3d89 kind: Event lastTimestamp: "2026-05-26T11:46:10Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:30bffa55f5e3e0e2f6d023c0f84846c206091122d724f5a0e810b4fb957dd389" already present on machine metadata: creationTimestamp: "2026-05-26T11:46:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:46:10Z" name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4.18b31b0173d459fe namespace: kserve-ci-e2e-test resourceVersion: "19717" uid: ed1904c6-bb01-4210-8452-7b97e06f9977 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4 namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 4eb67ebf-1d83-4854-8077-8ed885de3d89 kind: Event lastTimestamp: "2026-05-26T11:46:10Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T11:46:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:46:10Z" name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4.18b31b0176838f24 namespace: kserve-ci-e2e-test resourceVersion: "19718" uid: b38d6adf-f16a-452d-9f25-7849d2888828 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4 namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 4eb67ebf-1d83-4854-8077-8ed885de3d89 kind: Event lastTimestamp: "2026-05-26T11:46:10Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T11:46:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:46:10Z" name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4.18b31b0176f864a0 namespace: kserve-ci-e2e-test resourceVersion: "19719" uid: 17ac1258-4bdc-45c2-bf05-112531fb02d9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4 namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 4eb67ebf-1d83-4854-8077-8ed885de3d89 kind: Event lastTimestamp: "2026-05-26T11:46:15Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T11:46:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:46:15Z" name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4.18b31b0288e7edac namespace: kserve-ci-e2e-test resourceVersion: "19758" uid: d0768500-d1b2-453e-9a80-d035cace0a83 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4 namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 4eb67ebf-1d83-4854-8077-8ed885de3d89 kind: Event lastTimestamp: "2026-05-26T11:46:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:46:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:46:15Z" name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4.18b31b028be7b94a namespace: kserve-ci-e2e-test resourceVersion: "19761" uid: 4230dd34-25e0-4571-ae45-e5871dc3c88b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4 namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 4eb67ebf-1d83-4854-8077-8ed885de3d89 kind: Event lastTimestamp: "2026-05-26T11:46:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:46:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:46:15Z" name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4.18b31b028c53d8d2 namespace: kserve-ci-e2e-test resourceVersion: "19763" uid: 81e2d481-388b-4206-8aa8-1b38d14bdc30 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4 namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 4eb67ebf-1d83-4854-8077-8ed885de3d89 kind: Event lastTimestamp: "2026-05-26T11:46:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T11:46:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:46:15Z" name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4.18b31b028c5f30bc namespace: kserve-ci-e2e-test resourceVersion: "19765" uid: 76999509-821f-4dc9-bc20-bc34bc35e0a5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4 namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 4eb67ebf-1d83-4854-8077-8ed885de3d89 kind: Event lastTimestamp: "2026-05-26T11:46:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:46:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:46:15Z" name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4.18b31b028ed3f604 namespace: kserve-ci-e2e-test resourceVersion: "19768" uid: c9a0f13d-7b50-4bf9-981a-eaa259023f7b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4 namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 4eb67ebf-1d83-4854-8077-8ed885de3d89 kind: Event lastTimestamp: "2026-05-26T11:46:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:46:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:46:15Z" name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4.18b31b028f3e8f01 namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: 74aabafa-6ee2-4f0f-a853-41cd2e91f0a1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-26T11:46:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4 namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 4eb67ebf-1d83-4854-8077-8ed885de3d89 kind: Event lastTimestamp: "2026-05-26T11:47:32Z" message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:46:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:47:32Z" name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4.18b31b02c4d3f49d namespace: kserve-ci-e2e-test resourceVersion: "20391" uid: e17caf32-4e74-4d72-9071-2cec37c933ac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:47:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4 namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 4eb67ebf-1d83-4854-8077-8ed885de3d89 kind: Event lastTimestamp: "2026-05-26T11:47:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T11:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:47:30Z" name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4.18b31b13ffb1093b namespace: kserve-ci-e2e-test resourceVersion: "20335" uid: fb8e9af7-53a3-4102-b197-3679147812f9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:47:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4 namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 4eb67ebf-1d83-4854-8077-8ed885de3d89 kind: Event lastTimestamp: "2026-05-26T11:47:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:47:30Z" name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4.18b31b13ffb16f6c namespace: kserve-ci-e2e-test resourceVersion: "20338" uid: 2ad96c7a-9fbd-4538-8ab2-998e42251d18 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:47:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4 namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 4eb67ebf-1d83-4854-8077-8ed885de3d89 kind: Event lastTimestamp: "2026-05-26T11:47:32Z" message: 'Readiness probe failed: Get "https://10.132.0.28:8643/healthz": dial tcp 10.132.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:47:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:47:32Z" name: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4.18b31b1476ea5aa5 namespace: kserve-ci-e2e-test resourceVersion: "20390" uid: 9794fa3c-ed43-45d0-be8e-30bfed145c42 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-4e4c6-predictor-6cb9cfb575 namespace: kserve-ci-e2e-test resourceVersion: "19680" uid: 6b5f1631-d1f5-4f9d-8c7a-08cd24739a2c kind: Event lastTimestamp: "2026-05-26T11:46:10Z" message: 'Created pod: raw-sklearn-4e4c6-predictor-6cb9cfb575-qsmj4' metadata: creationTimestamp: "2026-05-26T11:46:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:46:10Z" name: raw-sklearn-4e4c6-predictor-6cb9cfb575.18b31b01591400b0 namespace: kserve-ci-e2e-test resourceVersion: "19696" uid: cab2d304-e894-413a-970c-a6aec2f2b016 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-4e4c6-predictor namespace: kserve-ci-e2e-test resourceVersion: "19678" uid: 0cd40cd3-63f5-472d-81a3-52ae4bb3b6f6 kind: Event lastTimestamp: "2026-05-26T11:46:10Z" message: Scaled up replica set raw-sklearn-4e4c6-predictor-6cb9cfb575 from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:46:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:46:10Z" name: raw-sklearn-4e4c6-predictor.18b31b0155687d4b namespace: kserve-ci-e2e-test resourceVersion: "19683" uid: 49ac972a-f50d-4f51-96d4-999ee1ca6b1d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:46:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-4e4c6-predictor namespace: kserve-ci-e2e-test resourceVersion: "19679" uid: 50230df7-3e42-41ef-bc9e-9cec72e4e7f4 kind: Event lastTimestamp: "2026-05-26T11:46:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:46:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:46:40Z" name: raw-sklearn-4e4c6-predictor.18b31b04d3e2a27a namespace: kserve-ci-e2e-test resourceVersion: "19945" uid: 11fbf04b-3ed7-4a63-ae17-508ecfe68507 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:46:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-4e4c6-predictor namespace: kserve-ci-e2e-test resourceVersion: "19679" uid: 50230df7-3e42-41ef-bc9e-9cec72e4e7f4 kind: Event lastTimestamp: "2026-05-26T11:46:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:46:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:46:40Z" name: raw-sklearn-4e4c6-predictor.18b31b04d460a856 namespace: kserve-ci-e2e-test resourceVersion: "19946" uid: 790f27a7-bb3d-41bc-8de6-e98b8104a666 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:46:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-4e4c6-predictor namespace: kserve-ci-e2e-test resourceVersion: "19841" uid: 50230df7-3e42-41ef-bc9e-9cec72e4e7f4 kind: Event lastTimestamp: "2026-05-26T11:47:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:47:25Z" name: raw-sklearn-4e4c6-predictor.18b31b0bd2e3e08a namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: 8c68de4b-2d9a-43df-976d-fe4c3cfa8caa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:46:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-4e4c6-predictor namespace: kserve-ci-e2e-test resourceVersion: "19841" uid: 50230df7-3e42-41ef-bc9e-9cec72e4e7f4 kind: Event lastTimestamp: "2026-05-26T11:47:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:47:25Z" name: raw-sklearn-4e4c6-predictor.18b31b0bd359b0bc namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 9e79b929-4e89-493a-ba5a-41e90d89d345 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-05-26T11:46:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-4e4c6 namespace: kserve-ci-e2e-test resourceVersion: "19666" uid: 71e212b7-5c6a-4861-8983-393a3a2d7a05 kind: Event lastTimestamp: "2026-05-26T11:47:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:46:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:47:22Z" name: raw-sklearn-4e4c6.18b31b0155342d44 namespace: kserve-ci-e2e-test resourceVersion: "20249" uid: e5d336b6-1fad-4038-be8b-2ab38110bdbe reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:46:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-4e4c6 namespace: kserve-ci-e2e-test resourceVersion: "19666" uid: 71e212b7-5c6a-4861-8983-393a3a2d7a05 kind: Event lastTimestamp: "2026-05-26T11:46:10Z" message: 'Failed to update status for InferenceService "raw-sklearn-4e4c6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-4e4c6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:46:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:46:10Z" name: raw-sklearn-4e4c6.18b31b0155c0776b namespace: kserve-ci-e2e-test resourceVersion: "19706" uid: d6eb4a86-4e51-4213-b824-53f769ae4142 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-4e4c6 namespace: kserve-ci-e2e-test resourceVersion: "19688" uid: 71e212b7-5c6a-4861-8983-393a3a2d7a05 kind: Event lastTimestamp: "2026-05-26T11:46:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-4e4c6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:46:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:46:10Z" name: raw-sklearn-4e4c6.18b31b015d7e837b namespace: kserve-ci-e2e-test resourceVersion: "19707" uid: e40f61b8-ee84-47f0-b63f-f05a75b5039c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:47:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-4e4c6 namespace: kserve-ci-e2e-test resourceVersion: "20247" uid: 71e212b7-5c6a-4861-8983-393a3a2d7a05 kind: Event lastTimestamp: "2026-05-26T11:47:22Z" message: InferenceService [raw-sklearn-4e4c6] is Ready metadata: creationTimestamp: "2026-05-26T11:47:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:47:22Z" name: raw-sklearn-4e4c6.18b31b1227e76224 namespace: kserve-ci-e2e-test resourceVersion: "20248" uid: 540f8cad-2586-4ee9-b10e-08b949b42641 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:47:30.532019Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj namespace: kserve-ci-e2e-test resourceVersion: "20352" uid: 35c1c88b-4d98-42c3-86b5-2895bbb980fc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj to ip-10-0-132-203.ec2.internal metadata: creationTimestamp: "2026-05-26T11:47:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:47:30Z" name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj.18b31b14049acf04 namespace: kserve-ci-e2e-test resourceVersion: "20357" uid: 053cc23c-2ffe-4b80-b2ce-a4c94c32c91b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-cfd45b8bd-csfvl source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:47:30Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj namespace: kserve-ci-e2e-test resourceVersion: "20361" uid: 35c1c88b-4d98-42c3-86b5-2895bbb980fc kind: Event lastTimestamp: "2026-05-26T11:47:30Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:47:30Z" name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj.18b31b141d8e6447 namespace: kserve-ci-e2e-test resourceVersion: "20369" uid: 2eb8aa86-9201-40be-98c5-f1d1c4664a80 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:47:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 35c1c88b-4d98-42c3-86b5-2895bbb980fc kind: Event lastTimestamp: "2026-05-26T11:47:30Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:30bffa55f5e3e0e2f6d023c0f84846c206091122d724f5a0e810b4fb957dd389" already present on machine metadata: creationTimestamp: "2026-05-26T11:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:47:30Z" name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj.18b31b141f154043 namespace: kserve-ci-e2e-test resourceVersion: "20371" uid: 15dc681d-56fc-40be-a88d-0c1678290730 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:47:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 35c1c88b-4d98-42c3-86b5-2895bbb980fc kind: Event lastTimestamp: "2026-05-26T11:47:31Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T11:47:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:47:31Z" name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj.18b31b1421a3e4b1 namespace: kserve-ci-e2e-test resourceVersion: "20373" uid: ba4ea62d-1e76-4ca7-888d-23c888e74214 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:47:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 35c1c88b-4d98-42c3-86b5-2895bbb980fc kind: Event lastTimestamp: "2026-05-26T11:47:31Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T11:47:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:47:31Z" name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj.18b31b14225755c6 namespace: kserve-ci-e2e-test resourceVersion: "20374" uid: 6969821d-7215-4d74-ba98-cc0380e93ace reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 35c1c88b-4d98-42c3-86b5-2895bbb980fc kind: Event lastTimestamp: "2026-05-26T11:47:35Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T11:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:47:35Z" name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj.18b31b1536e313de namespace: kserve-ci-e2e-test resourceVersion: "20420" uid: 1f076869-2f33-491a-a01e-1a9ef631be48 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 35c1c88b-4d98-42c3-86b5-2895bbb980fc kind: Event lastTimestamp: "2026-05-26T11:47:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:47:35Z" name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj.18b31b1539c9cf49 namespace: kserve-ci-e2e-test resourceVersion: "20422" uid: 932d5156-45a8-4883-9b05-9b02f7bca8dd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 35c1c88b-4d98-42c3-86b5-2895bbb980fc kind: Event lastTimestamp: "2026-05-26T11:47:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:47:35Z" name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj.18b31b153a3dfcb3 namespace: kserve-ci-e2e-test resourceVersion: "20423" uid: ddd30080-fd1c-4531-b38e-29b9743efbee reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 35c1c88b-4d98-42c3-86b5-2895bbb980fc kind: Event lastTimestamp: "2026-05-26T11:47:35Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T11:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:47:35Z" name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj.18b31b153a64b532 namespace: kserve-ci-e2e-test resourceVersion: "20426" uid: fe401597-8131-401e-8609-676b00042fe5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 35c1c88b-4d98-42c3-86b5-2895bbb980fc kind: Event lastTimestamp: "2026-05-26T11:47:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:47:35Z" name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj.18b31b153d65056f namespace: kserve-ci-e2e-test resourceVersion: "20430" uid: 2befb03a-2c62-4b1b-8b63-c1c6ca8411d3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 35c1c88b-4d98-42c3-86b5-2895bbb980fc kind: Event lastTimestamp: "2026-05-26T11:47:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:47:35Z" name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj.18b31b153dd5d27e namespace: kserve-ci-e2e-test resourceVersion: "20431" uid: a9aecdb3-96c9-45d2-8450-328a965bb9fd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-26T11:47:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 35c1c88b-4d98-42c3-86b5-2895bbb980fc kind: Event lastTimestamp: "2026-05-26T11:48:52Z" message: 'Readiness probe failed: dial tcp 10.132.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:47:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:48:52Z" name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj.18b31b1572dd170d namespace: kserve-ci-e2e-test resourceVersion: "21047" uid: 827efd14-2f1e-4ad5-9231-b3812bc16582 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:48:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 35c1c88b-4d98-42c3-86b5-2895bbb980fc kind: Event lastTimestamp: "2026-05-26T11:48:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T11:48:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:48:50Z" name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj.18b31b26aa354944 namespace: kserve-ci-e2e-test resourceVersion: "20990" uid: 066cb204-57f3-4417-804e-24ddde0a8416 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:48:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 35c1c88b-4d98-42c3-86b5-2895bbb980fc kind: Event lastTimestamp: "2026-05-26T11:48:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:48:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:48:50Z" name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj.18b31b26aa3604a7 namespace: kserve-ci-e2e-test resourceVersion: "20991" uid: fbeb246c-d281-4186-af49-2221a6e50b5a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:48:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 35c1c88b-4d98-42c3-86b5-2895bbb980fc kind: Event lastTimestamp: "2026-05-26T11:48:52Z" message: 'Readiness probe failed: Get "https://10.132.0.29:8643/healthz": dial tcp 10.132.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:48:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:48:52Z" name: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj.18b31b2724f86a6d namespace: kserve-ci-e2e-test resourceVersion: "21046" uid: e45f86eb-5c27-44fa-928d-116446f7904f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-203.ec2.internal source: component: kubelet host: ip-10-0-132-203.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:47:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-51723-predictor-9f4779f89 namespace: kserve-ci-e2e-test resourceVersion: "20341" uid: 81dca528-9fe5-44af-a0a3-ee360b7bfb31 kind: Event lastTimestamp: "2026-05-26T11:47:30Z" message: 'Created pod: raw-sklearn-runtime-51723-predictor-9f4779f89-p72pj' metadata: creationTimestamp: "2026-05-26T11:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:47:30Z" name: raw-sklearn-runtime-51723-predictor-9f4779f89.18b31b1404243979 namespace: kserve-ci-e2e-test resourceVersion: "20354" uid: 00f5cdec-dba8-4f71-90fb-5e5c05cf6ae1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:47:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-51723-predictor namespace: kserve-ci-e2e-test resourceVersion: "20339" uid: 9fbed7fa-3f69-47eb-a29c-50b05d29e050 kind: Event lastTimestamp: "2026-05-26T11:47:30Z" message: Scaled up replica set raw-sklearn-runtime-51723-predictor-9f4779f89 from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:47:30Z" name: raw-sklearn-runtime-51723-predictor.18b31b1400ef1515 namespace: kserve-ci-e2e-test resourceVersion: "20343" uid: 1c373adc-6f90-43d4-93ab-041d76c40371 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:47:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-51723-predictor namespace: kserve-ci-e2e-test resourceVersion: "20340" uid: d6a35f77-21c2-4c12-b37d-149de152f134 kind: Event lastTimestamp: "2026-05-26T11:48:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:47:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:48:15Z" name: raw-sklearn-runtime-51723-predictor.18b31b17800ec987 namespace: kserve-ci-e2e-test resourceVersion: "20720" uid: b0d7004f-a0f7-49d5-9fd5-091b2732a8f5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:47:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-51723-predictor namespace: kserve-ci-e2e-test resourceVersion: "20340" uid: d6a35f77-21c2-4c12-b37d-149de152f134 kind: Event lastTimestamp: "2026-05-26T11:48:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:47:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:48:15Z" name: raw-sklearn-runtime-51723-predictor.18b31b17807ded32 namespace: kserve-ci-e2e-test resourceVersion: "20721" uid: b2328999-8a8d-45c6-bc21-251b94bd1eda reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:48:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-51723-predictor namespace: kserve-ci-e2e-test resourceVersion: "20504" uid: d6a35f77-21c2-4c12-b37d-149de152f134 kind: Event lastTimestamp: "2026-05-26T11:48:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:48:45Z" name: raw-sklearn-runtime-51723-predictor.18b31b1aff505bb3 namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: 0cca64bc-93e5-44cc-8810-30fb282b93ea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:48:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-51723-predictor namespace: kserve-ci-e2e-test resourceVersion: "20504" uid: d6a35f77-21c2-4c12-b37d-149de152f134 kind: Event lastTimestamp: "2026-05-26T11:48:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:48:45Z" name: raw-sklearn-runtime-51723-predictor.18b31b1afffab255 namespace: kserve-ci-e2e-test resourceVersion: "20941" uid: 9481aea1-99d6-4f33-b998-8cb58b2a7335 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-05-26T11:47:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-51723 namespace: kserve-ci-e2e-test resourceVersion: "20311" uid: c3e4033a-f783-4697-b037-16c6f5c5b787 kind: Event lastTimestamp: "2026-05-26T11:48:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:48:42Z" name: raw-sklearn-runtime-51723.18b31b1400cd13bd namespace: kserve-ci-e2e-test resourceVersion: "20920" uid: e22bdbe6-16a4-4a5b-9de6-66bee5ffa7ac reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:47:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-51723 namespace: kserve-ci-e2e-test resourceVersion: "20311" uid: c3e4033a-f783-4697-b037-16c6f5c5b787 kind: Event lastTimestamp: "2026-05-26T11:47:30Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-51723": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-51723": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:47:30Z" name: raw-sklearn-runtime-51723.18b31b1401358c10 namespace: kserve-ci-e2e-test resourceVersion: "20345" uid: ae344756-a090-478e-aff9-d08a87c9c751 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:48:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-51723 namespace: kserve-ci-e2e-test resourceVersion: "20917" uid: c3e4033a-f783-4697-b037-16c6f5c5b787 kind: Event lastTimestamp: "2026-05-26T11:48:42Z" message: InferenceService [raw-sklearn-runtime-51723] is Ready metadata: creationTimestamp: "2026-05-26T11:48:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:48:42Z" name: raw-sklearn-runtime-51723.18b31b24d701c80d namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: afabbf37-d318-4716-899b-cef6eb745c83 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "21204"