--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:40:54.547105Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-02b5ab-predictor-594949977c-bw5rb namespace: kserve-ci-e2e-test resourceVersion: "20599" uid: be581cf6-68df-41db-8a1a-e8a269a742fe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-02b5ab-predictor-594949977c-bw5rb to ip-10-0-135-227.ec2.internal metadata: creationTimestamp: "2026-07-02T21:40:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:40:54Z" name: isvc-init-fail-02b5ab-predictor-594949977c-bw5rb.18be96eeb9b1cdc9 namespace: kserve-ci-e2e-test resourceVersion: "20603" uid: 6a9a6380-1a4e-4ebf-a7e5-05901861db18 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-558f78ddf7-rgj7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:54Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-02b5ab-predictor-594949977c-bw5rb namespace: kserve-ci-e2e-test resourceVersion: "20609" uid: be581cf6-68df-41db-8a1a-e8a269a742fe kind: Event lastTimestamp: "2026-07-02T21:40:54Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:40:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:40:54Z" name: isvc-init-fail-02b5ab-predictor-594949977c-bw5rb.18be96eed2f4b47f namespace: kserve-ci-e2e-test resourceVersion: "20619" uid: cbb34747-bc41-4008-9400-0faf1d294024 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:40:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-02b5ab-predictor-594949977c-bw5rb namespace: kserve-ci-e2e-test resourceVersion: "20600" uid: be581cf6-68df-41db-8a1a-e8a269a742fe kind: Event lastTimestamp: "2026-07-02T21:40:58Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:33f82c067d520fe749f73749cf76b5935fe69bd9eb141504c6660698c7947743" already present on machine metadata: creationTimestamp: "2026-07-02T21:40:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:58Z" name: isvc-init-fail-02b5ab-predictor-594949977c-bw5rb.18be96eed45aada5 namespace: kserve-ci-e2e-test resourceVersion: "20652" uid: 5c4e30a3-5af5-4c34-87e7-1780e9f80c44 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:40:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-02b5ab-predictor-594949977c-bw5rb namespace: kserve-ci-e2e-test resourceVersion: "20600" uid: be581cf6-68df-41db-8a1a-e8a269a742fe kind: Event lastTimestamp: "2026-07-02T21:40:58Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T21:40:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:58Z" name: isvc-init-fail-02b5ab-predictor-594949977c-bw5rb.18be96eed7510645 namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 42d032c4-94bf-41b3-a566-aa1628b7129e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:40:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-02b5ab-predictor-594949977c-bw5rb namespace: kserve-ci-e2e-test resourceVersion: "20600" uid: be581cf6-68df-41db-8a1a-e8a269a742fe kind: Event lastTimestamp: "2026-07-02T21:40:58Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T21:40:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:58Z" name: isvc-init-fail-02b5ab-predictor-594949977c-bw5rb.18be96eed7bbf47e namespace: kserve-ci-e2e-test resourceVersion: "20656" uid: 71b71037-df5f-45a1-aea1-0223e4723ab1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-02b5ab-predictor-594949977c-bw5rb namespace: kserve-ci-e2e-test resourceVersion: "20600" uid: be581cf6-68df-41db-8a1a-e8a269a742fe kind: Event lastTimestamp: "2026-07-02T21:40:59Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-07-02T21:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:59Z" name: isvc-init-fail-02b5ab-predictor-594949977c-bw5rb.18be96efe27cdd85 namespace: kserve-ci-e2e-test resourceVersion: "20684" uid: 8edc45a2-cda8-4c9e-b8c2-38ff21aefedc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-02b5ab-predictor-594949977c namespace: kserve-ci-e2e-test resourceVersion: "20590" uid: e813e6c4-3fc8-4996-9be8-a74452716d54 kind: Event lastTimestamp: "2026-07-02T21:40:54Z" message: 'Created pod: isvc-init-fail-02b5ab-predictor-594949977c-bw5rb' metadata: creationTimestamp: "2026-07-02T21:40:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:40:54Z" name: isvc-init-fail-02b5ab-predictor-594949977c.18be96eeb913de78 namespace: kserve-ci-e2e-test resourceVersion: "20601" uid: 87c9c576-ccfb-4c38-b9f0-60f6a6d6b382 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-02b5ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "20588" uid: 07d3a741-3a6a-4e65-b661-dd5ca83c43e6 kind: Event lastTimestamp: "2026-07-02T21:40:54Z" message: Scaled up replica set isvc-init-fail-02b5ab-predictor-594949977c from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:40:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:40:54Z" name: isvc-init-fail-02b5ab-predictor.18be96eeb5adbb63 namespace: kserve-ci-e2e-test resourceVersion: "20591" uid: d76e97ce-aadc-47f3-8d2d-0e3789ac2dc5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T21:40:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-02b5ab namespace: kserve-ci-e2e-test resourceVersion: "20581" uid: 368530bc-8e90-42c0-9ec0-e4e57768a812 kind: Event lastTimestamp: "2026-07-02T21:40:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:40:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:40:58Z" name: isvc-init-fail-02b5ab.18be96eeb5b3e342 namespace: kserve-ci-e2e-test resourceVersion: "20659" uid: 75144559-19c8-47ad-a6c5-a8b2c886a46b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-02b5ab namespace: kserve-ci-e2e-test resourceVersion: "20581" uid: 368530bc-8e90-42c0-9ec0-e4e57768a812 kind: Event lastTimestamp: "2026-07-02T21:40:54Z" message: 'Failed to update status for InferenceService "isvc-init-fail-02b5ab": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-02b5ab": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:40:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:40:54Z" name: isvc-init-fail-02b5ab.18be96eeb657b890 namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: 26ea499c-b30f-4406-8dd0-800d99a6847c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-02b5ab namespace: kserve-ci-e2e-test resourceVersion: "20581" uid: 368530bc-8e90-42c0-9ec0-e4e57768a812 kind: Event lastTimestamp: "2026-07-02T21:40:54Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-02b5ab": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:40:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:40:54Z" name: isvc-init-fail-02b5ab.18be96eeb657fc7f namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: e2210f07-2b1c-4e92-b4f9-551f90203f9e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:34:52.502603Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6 namespace: kserve-ci-e2e-test resourceVersion: "17631" uid: 3aadcb6a-96b7-456e-8eb7-2b710f299456 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6 to ip-10-0-135-227.ec2.internal metadata: creationTimestamp: "2026-07-02T21:34:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:34:52Z" name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6.18be969a6e291b96 namespace: kserve-ci-e2e-test resourceVersion: "17635" uid: 17bd9d29-4a43-4cc8-8769-8ef4581574d5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-558f78ddf7-rgj7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6 namespace: kserve-ci-e2e-test resourceVersion: "17640" uid: 3aadcb6a-96b7-456e-8eb7-2b710f299456 kind: Event lastTimestamp: "2026-07-02T21:34:52Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:34:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:34:52Z" name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6.18be969a8790f3d2 namespace: kserve-ci-e2e-test resourceVersion: "17648" uid: 6de6c296-1c6b-4d25-9697-87b3441dfc4a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6 namespace: kserve-ci-e2e-test resourceVersion: "17633" uid: 3aadcb6a-96b7-456e-8eb7-2b710f299456 kind: Event lastTimestamp: "2026-07-02T21:34:52Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:33f82c067d520fe749f73749cf76b5935fe69bd9eb141504c6660698c7947743" already present on machine metadata: creationTimestamp: "2026-07-02T21:34:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:52Z" name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6.18be969a890590ea namespace: kserve-ci-e2e-test resourceVersion: "17650" uid: 57927c3a-78b9-49d7-a530-1f1fdca85cc8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6 namespace: kserve-ci-e2e-test resourceVersion: "17633" uid: 3aadcb6a-96b7-456e-8eb7-2b710f299456 kind: Event lastTimestamp: "2026-07-02T21:34:52Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T21:34:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:52Z" name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6.18be969a8ba08a01 namespace: kserve-ci-e2e-test resourceVersion: "17652" uid: 018afb41-51f2-4dfe-8175-ed0e81b08a12 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6 namespace: kserve-ci-e2e-test resourceVersion: "17633" uid: 3aadcb6a-96b7-456e-8eb7-2b710f299456 kind: Event lastTimestamp: "2026-07-02T21:34:53Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T21:34:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:53Z" name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6.18be969a8c5abdb6 namespace: kserve-ci-e2e-test resourceVersion: "17653" uid: d20ee931-e52e-40e0-a5cf-913ac11432f4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6 namespace: kserve-ci-e2e-test resourceVersion: "17633" uid: 3aadcb6a-96b7-456e-8eb7-2b710f299456 kind: Event lastTimestamp: "2026-07-02T21:34:57Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:34:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:57Z" name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6.18be969b7c4adf77 namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: 9e25521e-70fb-4944-8250-cab9285b94ca reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6 namespace: kserve-ci-e2e-test resourceVersion: "17633" uid: 3aadcb6a-96b7-456e-8eb7-2b710f299456 kind: Event lastTimestamp: "2026-07-02T21:34:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:34:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:57Z" name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6.18be969b7f3c7ea4 namespace: kserve-ci-e2e-test resourceVersion: "17690" uid: df44d225-89ab-4c33-ae6e-2f656bf593f9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6 namespace: kserve-ci-e2e-test resourceVersion: "17633" uid: 3aadcb6a-96b7-456e-8eb7-2b710f299456 kind: Event lastTimestamp: "2026-07-02T21:34:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:34:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:57Z" name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6.18be969b7faa8f3a namespace: kserve-ci-e2e-test resourceVersion: "17692" uid: f3a2b180-950d-458e-ae94-14154880c9db reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6 namespace: kserve-ci-e2e-test resourceVersion: "17633" uid: 3aadcb6a-96b7-456e-8eb7-2b710f299456 kind: Event lastTimestamp: "2026-07-02T21:34:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:34:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:57Z" name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6.18be969b7fb4460e namespace: kserve-ci-e2e-test resourceVersion: "17694" uid: baa4f3a3-ff1c-4e70-a22a-01b18dd774ae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6 namespace: kserve-ci-e2e-test resourceVersion: "17633" uid: 3aadcb6a-96b7-456e-8eb7-2b710f299456 kind: Event lastTimestamp: "2026-07-02T21:34:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:34:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:57Z" name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6.18be969b825ccedd namespace: kserve-ci-e2e-test resourceVersion: "17696" uid: 067b80f5-c27d-418e-8ec6-1edce8ea4b7d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6 namespace: kserve-ci-e2e-test resourceVersion: "17633" uid: 3aadcb6a-96b7-456e-8eb7-2b710f299456 kind: Event lastTimestamp: "2026-07-02T21:34:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:34:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:57Z" name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6.18be969b82c9bc14 namespace: kserve-ci-e2e-test resourceVersion: "17698" uid: 6f6479ee-ec18-4420-bc88-71a1bf212828 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6 namespace: kserve-ci-e2e-test resourceVersion: "17633" uid: 3aadcb6a-96b7-456e-8eb7-2b710f299456 kind: Event lastTimestamp: "2026-07-02T21:34:57Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:7fce5049d5e22f5c18756419ab95335e509bf71d40b49159f51a66f5ab33232f" already present on machine metadata: creationTimestamp: "2026-07-02T21:34:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:57Z" name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6.18be969b82d28b44 namespace: kserve-ci-e2e-test resourceVersion: "17699" uid: b48d02c0-aa3f-42bc-9f4e-78112bc4bfbc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6 namespace: kserve-ci-e2e-test resourceVersion: "17633" uid: 3aadcb6a-96b7-456e-8eb7-2b710f299456 kind: Event lastTimestamp: "2026-07-02T21:34:57Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-02T21:34:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:57Z" name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6.18be969b8588f186 namespace: kserve-ci-e2e-test resourceVersion: "17701" uid: 091eddd8-c6ad-4d41-b075-baef2b96872d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6 namespace: kserve-ci-e2e-test resourceVersion: "17633" uid: 3aadcb6a-96b7-456e-8eb7-2b710f299456 kind: Event lastTimestamp: "2026-07-02T21:34:57Z" message: Started container agent metadata: creationTimestamp: "2026-07-02T21:34:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:57Z" name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6.18be969b85f7b311 namespace: kserve-ci-e2e-test resourceVersion: "17702" uid: 51aa97e0-6e86-499f-b3e9-324b2582a074 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T21:34:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6 namespace: kserve-ci-e2e-test resourceVersion: "17633" uid: 3aadcb6a-96b7-456e-8eb7-2b710f299456 kind: Event lastTimestamp: "2026-07-02T21:36:43Z" message: 'Readiness probe failed: dial tcp 10.132.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:34:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:36:43Z" name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6.18be969bb85d3833 namespace: kserve-ci-e2e-test resourceVersion: "18597" uid: 77fb4da6-9f4b-4263-8a3e-6a43590843e1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T21:34:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6 namespace: kserve-ci-e2e-test resourceVersion: "17633" uid: 3aadcb6a-96b7-456e-8eb7-2b710f299456 kind: Event lastTimestamp: "2026-07-02T21:36:33Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T21:34:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:36:33Z" name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6.18be969bf4241fbf namespace: kserve-ci-e2e-test resourceVersion: "18507" uid: 4ce343ba-c463-4c26-b952-424b12149662 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:36:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6 namespace: kserve-ci-e2e-test resourceVersion: "17633" uid: 3aadcb6a-96b7-456e-8eb7-2b710f299456 kind: Event lastTimestamp: "2026-07-02T21:36:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:36:27Z" name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6.18be96b09f8a5566 namespace: kserve-ci-e2e-test resourceVersion: "18422" uid: 2be4f8e1-2f3b-4a10-8f2c-cb0c96465982 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:36:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6 namespace: kserve-ci-e2e-test resourceVersion: "17633" uid: 3aadcb6a-96b7-456e-8eb7-2b710f299456 kind: Event lastTimestamp: "2026-07-02T21:36:27Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-02T21:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:36:27Z" name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6.18be96b09f8a96c6 namespace: kserve-ci-e2e-test resourceVersion: "18424" uid: 95ccb131-a876-42a4-98cd-742043e14e01 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:36:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6 namespace: kserve-ci-e2e-test resourceVersion: "17633" uid: 3aadcb6a-96b7-456e-8eb7-2b710f299456 kind: Event lastTimestamp: "2026-07-02T21:36:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:36:27Z" name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6.18be96b09f8ae6d6 namespace: kserve-ci-e2e-test resourceVersion: "18427" uid: dc13ce0e-2506-4f1e-9c42-a7b78c63a8a9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:36:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6 namespace: kserve-ci-e2e-test resourceVersion: "17633" uid: 3aadcb6a-96b7-456e-8eb7-2b710f299456 kind: Event lastTimestamp: "2026-07-02T21:36:40Z" message: 'Readiness probe failed: Get "https://10.132.0.29:8643/healthz": dial tcp 10.132.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:36:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:36:40Z" name: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6.18be96b124608718 namespace: kserve-ci-e2e-test resourceVersion: "18570" uid: 04d28efd-6728-4bfd-b049-8c27a170935a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-1f359-predictor-5778d849d6 namespace: kserve-ci-e2e-test resourceVersion: "17618" uid: c366840f-8ec8-43d2-847a-161a8ec8ce1d kind: Event lastTimestamp: "2026-07-02T21:34:52Z" message: 'Created pod: isvc-logger-raw-1f359-predictor-5778d849d6-2mwz6' metadata: creationTimestamp: "2026-07-02T21:34:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:34:52Z" name: isvc-logger-raw-1f359-predictor-5778d849d6.18be969a6db465fe namespace: kserve-ci-e2e-test resourceVersion: "17632" uid: 72fa0b34-a143-4378-9501-9d5d0022e579 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-1f359-predictor namespace: kserve-ci-e2e-test resourceVersion: "17616" uid: 167a8904-03c8-49c3-a744-48d0005fef38 kind: Event lastTimestamp: "2026-07-02T21:34:52Z" message: Scaled up replica set isvc-logger-raw-1f359-predictor-5778d849d6 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:34:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:34:52Z" name: isvc-logger-raw-1f359-predictor.18be969a69a66825 namespace: kserve-ci-e2e-test resourceVersion: "17620" uid: 37239ab9-41e8-439a-b70f-2f2a9e0d2157 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:35:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-1f359-predictor namespace: kserve-ci-e2e-test resourceVersion: "17617" uid: 7c78bb82-09e7-4da3-89e1-a05f5608b92b kind: Event lastTimestamp: "2026-07-02T21:35:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:35:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:35:37Z" name: isvc-logger-raw-1f359-predictor.18be969de8ba4c55 namespace: kserve-ci-e2e-test resourceVersion: "18013" uid: bfb03fee-0005-427e-a53f-7b52a49d20ca reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:35:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-1f359-predictor namespace: kserve-ci-e2e-test resourceVersion: "17617" uid: 7c78bb82-09e7-4da3-89e1-a05f5608b92b kind: Event lastTimestamp: "2026-07-02T21:35:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:35:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:35:37Z" name: isvc-logger-raw-1f359-predictor.18be969de918b179 namespace: kserve-ci-e2e-test resourceVersion: "18014" uid: 0abfcd02-414c-4a0e-8fe9-2f6f74507c37 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:35:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-1f359-predictor namespace: kserve-ci-e2e-test resourceVersion: "17788" uid: 7c78bb82-09e7-4da3-89e1-a05f5608b92b kind: Event lastTimestamp: "2026-07-02T21:36:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:35:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:36:22Z" name: isvc-logger-raw-1f359-predictor.18be96a167d5817c namespace: kserve-ci-e2e-test resourceVersion: "18339" uid: 21ffa75a-06e2-46c3-b09f-42774e746810 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:35:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-1f359-predictor namespace: kserve-ci-e2e-test resourceVersion: "17788" uid: 7c78bb82-09e7-4da3-89e1-a05f5608b92b kind: Event lastTimestamp: "2026-07-02T21:36:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:35:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:36:22Z" name: isvc-logger-raw-1f359-predictor.18be96a1683ff4e6 namespace: kserve-ci-e2e-test resourceVersion: "18340" uid: a6d49409-6354-4d74-ba19-f5b40fb20574 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T21:34:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-1f359 namespace: kserve-ci-e2e-test resourceVersion: "17608" uid: 50ad6306-298a-4604-a7c0-e92dbb62d684 kind: Event lastTimestamp: "2026-07-02T21:36:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:34:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:36:13Z" name: isvc-logger-raw-1f359.18be969a69bb5ec9 namespace: kserve-ci-e2e-test resourceVersion: "18275" uid: ed80c26f-8c92-4e6c-bf2c-7bb81b645c50 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:34:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-1f359 namespace: kserve-ci-e2e-test resourceVersion: "17608" uid: 50ad6306-298a-4604-a7c0-e92dbb62d684 kind: Event lastTimestamp: "2026-07-02T21:34:52Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-1f359": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-1f359": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:34:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:34:52Z" name: isvc-logger-raw-1f359.18be969a6a4b8572 namespace: kserve-ci-e2e-test resourceVersion: "17642" uid: 159916c2-deff-4ef7-a541-00a22c481d33 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-1f359 namespace: kserve-ci-e2e-test resourceVersion: "17613" uid: 50ad6306-298a-4604-a7c0-e92dbb62d684 kind: Event lastTimestamp: "2026-07-02T21:34:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-1f359": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:34:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:34:52Z" name: isvc-logger-raw-1f359.18be969a71077162 namespace: kserve-ci-e2e-test resourceVersion: "17643" uid: cf0da909-2113-4b20-a813-85142d750766 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:36:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-1f359 namespace: kserve-ci-e2e-test resourceVersion: "18272" uid: 50ad6306-298a-4604-a7c0-e92dbb62d684 kind: Event lastTimestamp: "2026-07-02T21:36:13Z" message: InferenceService [isvc-logger-raw-1f359] is Ready metadata: creationTimestamp: "2026-07-02T21:36:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:36:13Z" name: isvc-logger-raw-1f359.18be96ad32a7f2c5 namespace: kserve-ci-e2e-test resourceVersion: "18273" uid: 66a7ca38-b273-4aef-97e3-b1236e6c2b2e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:39:13.148110Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9 namespace: kserve-ci-e2e-test resourceVersion: "19700" uid: 6394bdea-c72b-4f71-878f-10c98235628a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9 to ip-10-0-135-227.ec2.internal metadata: creationTimestamp: "2026-07-02T21:39:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:39:13Z" name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9.18be96d71dd7e41a namespace: kserve-ci-e2e-test resourceVersion: "19705" uid: 7f5d7a26-b72a-4f79-b260-5a71325de64d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-558f78ddf7-rgj7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:39:13Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9 namespace: kserve-ci-e2e-test resourceVersion: "19710" uid: 6394bdea-c72b-4f71-878f-10c98235628a kind: Event lastTimestamp: "2026-07-02T21:39:13Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:39:13Z" name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9.18be96d737cce0f6 namespace: kserve-ci-e2e-test resourceVersion: "19719" uid: 6e840f01-28ae-4696-97b5-1681de45f7ab reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:39:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9 namespace: kserve-ci-e2e-test resourceVersion: "19703" uid: 6394bdea-c72b-4f71-878f-10c98235628a kind: Event lastTimestamp: "2026-07-02T21:39:13Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:33f82c067d520fe749f73749cf76b5935fe69bd9eb141504c6660698c7947743" already present on machine metadata: creationTimestamp: "2026-07-02T21:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:39:13Z" name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9.18be96d7393f2f53 namespace: kserve-ci-e2e-test resourceVersion: "19721" uid: 89f24372-bf54-42fa-a289-f0c4b77a6e95 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:39:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9 namespace: kserve-ci-e2e-test resourceVersion: "19703" uid: 6394bdea-c72b-4f71-878f-10c98235628a kind: Event lastTimestamp: "2026-07-02T21:39:13Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T21:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:39:13Z" name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9.18be96d73c1246b8 namespace: kserve-ci-e2e-test resourceVersion: "19722" uid: 981309d3-5eb6-43d6-8a93-2455e10c4409 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:39:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9 namespace: kserve-ci-e2e-test resourceVersion: "19703" uid: 6394bdea-c72b-4f71-878f-10c98235628a kind: Event lastTimestamp: "2026-07-02T21:39:13Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T21:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:39:13Z" name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9.18be96d73c8458f2 namespace: kserve-ci-e2e-test resourceVersion: "19723" uid: 40c8433c-8637-4ade-9c55-bba82ce62dc6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:39:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9 namespace: kserve-ci-e2e-test resourceVersion: "19703" uid: 6394bdea-c72b-4f71-878f-10c98235628a kind: Event lastTimestamp: "2026-07-02T21:39:18Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:39:18Z" name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9.18be96d844d52d6d namespace: kserve-ci-e2e-test resourceVersion: "19764" uid: 7c7cdcb0-71e8-4ce2-a2c8-f66029fc9177 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:39:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9 namespace: kserve-ci-e2e-test resourceVersion: "19703" uid: 6394bdea-c72b-4f71-878f-10c98235628a kind: Event lastTimestamp: "2026-07-02T21:39:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:39:18Z" name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9.18be96d847886996 namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: 6c514e3f-b923-46c3-9b8f-89ec01bf1e37 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:39:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9 namespace: kserve-ci-e2e-test resourceVersion: "19703" uid: 6394bdea-c72b-4f71-878f-10c98235628a kind: Event lastTimestamp: "2026-07-02T21:39:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:39:18Z" name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9.18be96d8481444c7 namespace: kserve-ci-e2e-test resourceVersion: "19770" uid: ecf6d4d6-7a41-4a2f-bd69-1e3784dde880 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:39:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9 namespace: kserve-ci-e2e-test resourceVersion: "19703" uid: 6394bdea-c72b-4f71-878f-10c98235628a kind: Event lastTimestamp: "2026-07-02T21:39:18Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:39:18Z" name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9.18be96d8481ebc40 namespace: kserve-ci-e2e-test resourceVersion: "19771" uid: 0d6c4d28-c0d2-4f4d-b72d-a6af447ca077 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:39:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9 namespace: kserve-ci-e2e-test resourceVersion: "19703" uid: 6394bdea-c72b-4f71-878f-10c98235628a kind: Event lastTimestamp: "2026-07-02T21:39:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:39:18Z" name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9.18be96d84b8cee73 namespace: kserve-ci-e2e-test resourceVersion: "19775" uid: fbbe7d77-adfb-4304-850f-c8f5a8a4dee0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:39:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9 namespace: kserve-ci-e2e-test resourceVersion: "19703" uid: 6394bdea-c72b-4f71-878f-10c98235628a kind: Event lastTimestamp: "2026-07-02T21:39:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:39:18Z" name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9.18be96d84bfacb66 namespace: kserve-ci-e2e-test resourceVersion: "19776" uid: 77603c88-a473-4c0c-8df6-ba02dd30d301 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T21:39:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9 namespace: kserve-ci-e2e-test resourceVersion: "19703" uid: 6394bdea-c72b-4f71-878f-10c98235628a kind: Event lastTimestamp: "2026-07-02T21:40:26Z" message: 'Readiness probe failed: dial tcp 10.132.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:39:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:26Z" name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9.18be96d8bcb6e207 namespace: kserve-ci-e2e-test resourceVersion: "20258" uid: 48c725b9-06ca-4741-b3b9-97e691f393e5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9 namespace: kserve-ci-e2e-test resourceVersion: "19703" uid: 6394bdea-c72b-4f71-878f-10c98235628a kind: Event lastTimestamp: "2026-07-02T21:40:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:40:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:49Z" name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9.18be96ed867270fd namespace: kserve-ci-e2e-test resourceVersion: "20528" uid: b9996f9c-191d-49a8-8881-53e7388a3a7a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9 namespace: kserve-ci-e2e-test resourceVersion: "19703" uid: 6394bdea-c72b-4f71-878f-10c98235628a kind: Event lastTimestamp: "2026-07-02T21:40:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:40:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:49Z" name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9.18be96ed8672de46 namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: d4529b14-4076-4d92-9844-38ea29f6f06b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9 namespace: kserve-ci-e2e-test resourceVersion: "19703" uid: 6394bdea-c72b-4f71-878f-10c98235628a kind: Event lastTimestamp: "2026-07-02T21:40:51Z" message: 'Readiness probe failed: Get "https://10.132.0.31:8643/healthz": dial tcp 10.132.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:40:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:51Z" name: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9.18be96edece33895 namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: ba04f702-2cac-4934-8f72-6280ae86f7db reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:39:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-0bbdb6-predictor-847fc894cb namespace: kserve-ci-e2e-test resourceVersion: "19691" uid: 3914177f-9dc1-4819-ba6b-a4c46c757d03 kind: Event lastTimestamp: "2026-07-02T21:39:13Z" message: 'Created pod: isvc-primary-0bbdb6-predictor-847fc894cb-g2zr9' metadata: creationTimestamp: "2026-07-02T21:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:39:13Z" name: isvc-primary-0bbdb6-predictor-847fc894cb.18be96d71d727d81 namespace: kserve-ci-e2e-test resourceVersion: "19702" uid: a793f385-6463-46ef-b915-281d7eea78e7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:39:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-0bbdb6-predictor namespace: kserve-ci-e2e-test resourceVersion: "19689" uid: e6eb69d2-189f-4f04-8d1c-f0a4ef3237a2 kind: Event lastTimestamp: "2026-07-02T21:39:13Z" message: Scaled up replica set isvc-primary-0bbdb6-predictor-847fc894cb from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:39:13Z" name: isvc-primary-0bbdb6-predictor.18be96d7192cda1e namespace: kserve-ci-e2e-test resourceVersion: "19692" uid: cc46998c-a4c8-4c08-9679-4a1b68b84b1f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:39:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-0bbdb6-predictor namespace: kserve-ci-e2e-test resourceVersion: "19690" uid: e8023aba-2ab6-4a2a-b102-414dac77d329 kind: Event lastTimestamp: "2026-07-02T21:39:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:39:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:39:43Z" name: isvc-primary-0bbdb6-predictor.18be96da97b899ce namespace: kserve-ci-e2e-test resourceVersion: "19951" uid: a1a4dbc9-d2a5-4810-9ed3-d3a1bba2fa57 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:39:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-0bbdb6-predictor namespace: kserve-ci-e2e-test resourceVersion: "19690" uid: e8023aba-2ab6-4a2a-b102-414dac77d329 kind: Event lastTimestamp: "2026-07-02T21:39:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:39:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:39:43Z" name: isvc-primary-0bbdb6-predictor.18be96da9820d310 namespace: kserve-ci-e2e-test resourceVersion: "19952" uid: f9cb14b5-93d6-4b99-9cfb-1752975129cc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:39:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-0bbdb6-predictor namespace: kserve-ci-e2e-test resourceVersion: "19850" uid: e8023aba-2ab6-4a2a-b102-414dac77d329 kind: Event lastTimestamp: "2026-07-02T21:40:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:40:43Z" name: isvc-primary-0bbdb6-predictor.18be96e19618d92e namespace: kserve-ci-e2e-test resourceVersion: "20451" uid: 9fc42d73-0966-4095-a6f5-d85dd1bf997c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:39:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-0bbdb6-predictor namespace: kserve-ci-e2e-test resourceVersion: "19850" uid: e8023aba-2ab6-4a2a-b102-414dac77d329 kind: Event lastTimestamp: "2026-07-02T21:40:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:39:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:40:43Z" name: isvc-primary-0bbdb6-predictor.18be96e1967ea2ec namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: 4d24a921-69ec-4f8d-b780-548d5d800309 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T21:39:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-0bbdb6 namespace: kserve-ci-e2e-test resourceVersion: "19682" uid: b078d92e-d9f7-4881-a400-b6c1f9e583b5 kind: Event lastTimestamp: "2026-07-02T21:40:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:40:28Z" name: isvc-primary-0bbdb6.18be96d71944cc64 namespace: kserve-ci-e2e-test resourceVersion: "20281" uid: f772139d-0acd-4360-a3e2-2a90332a4d54 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:39:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-0bbdb6 namespace: kserve-ci-e2e-test resourceVersion: "19682" uid: b078d92e-d9f7-4881-a400-b6c1f9e583b5 kind: Event lastTimestamp: "2026-07-02T21:39:13Z" message: 'Failed to update status for InferenceService "isvc-primary-0bbdb6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-0bbdb6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:39:13Z" name: isvc-primary-0bbdb6.18be96d719a4be0f namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 23df7724-a33f-45c5-9e82-393033138c90 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:39:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-0bbdb6 namespace: kserve-ci-e2e-test resourceVersion: "19682" uid: b078d92e-d9f7-4881-a400-b6c1f9e583b5 kind: Event lastTimestamp: "2026-07-02T21:39:13Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-0bbdb6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:39:13Z" name: isvc-primary-0bbdb6.18be96d719a513ad namespace: kserve-ci-e2e-test resourceVersion: "19696" uid: a9ce5a4e-a41d-4b73-8226-d4e39b5ad9f4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-0bbdb6 namespace: kserve-ci-e2e-test resourceVersion: "20277" uid: b078d92e-d9f7-4881-a400-b6c1f9e583b5 kind: Event lastTimestamp: "2026-07-02T21:40:28Z" message: InferenceService [isvc-primary-0bbdb6] is Ready metadata: creationTimestamp: "2026-07-02T21:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:40:28Z" name: isvc-primary-0bbdb6.18be96e891df8958 namespace: kserve-ci-e2e-test resourceVersion: "20279" uid: 64d059b7-b2ec-4223-a1d7-19a480044bc2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:28:56.809336Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn namespace: kserve-ci-e2e-test resourceVersion: "14425" uid: 3a4bb54c-cae0-4723-ad7c-aca4f345044e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn to ip-10-0-135-227.ec2.internal metadata: creationTimestamp: "2026-07-02T21:28:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:28:56Z" name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn.18be96479d3085f9 namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: ac92ebcd-631a-4d75-80ed-39113b77820d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-558f78ddf7-rgj7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:28:57Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn namespace: kserve-ci-e2e-test resourceVersion: "14435" uid: 3a4bb54c-cae0-4723-ad7c-aca4f345044e kind: Event lastTimestamp: "2026-07-02T21:28:57Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:28:57Z" name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn.18be9647b659ed33 namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: 57b159a3-0df4-4d8b-a14b-79884c9daef6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:28:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 3a4bb54c-cae0-4723-ad7c-aca4f345044e kind: Event lastTimestamp: "2026-07-02T21:28:57Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:33f82c067d520fe749f73749cf76b5935fe69bd9eb141504c6660698c7947743" already present on machine metadata: creationTimestamp: "2026-07-02T21:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:28:57Z" name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn.18be9647b7e09a3a namespace: kserve-ci-e2e-test resourceVersion: "14446" uid: d7944489-9dd6-4578-a98f-c37b0227b139 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:28:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 3a4bb54c-cae0-4723-ad7c-aca4f345044e kind: Event lastTimestamp: "2026-07-02T21:28:57Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T21:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:28:57Z" name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn.18be9647ba849c50 namespace: kserve-ci-e2e-test resourceVersion: "14447" uid: 0b2e8878-659b-4ef7-96dc-aae6003a75af reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:28:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 3a4bb54c-cae0-4723-ad7c-aca4f345044e kind: Event lastTimestamp: "2026-07-02T21:28:57Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T21:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:28:57Z" name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn.18be9647bb124b03 namespace: kserve-ci-e2e-test resourceVersion: "14448" uid: 205bda22-0cb4-4ec3-820d-198920b3df5d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 3a4bb54c-cae0-4723-ad7c-aca4f345044e kind: Event lastTimestamp: "2026-07-02T21:29:01Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" metadata: creationTimestamp: "2026-07-02T21:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:01Z" name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn.18be9648b33a4409 namespace: kserve-ci-e2e-test resourceVersion: "14483" uid: da0cbea3-bc92-4ac9-b7d4-778e97b7dc5c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 3a4bb54c-cae0-4723-ad7c-aca4f345044e kind: Event lastTimestamp: "2026-07-02T21:29:16Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" in 14.903s (14.903s including waiting). Image size: 1555748775 bytes.' metadata: creationTimestamp: "2026-07-02T21:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:16Z" name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn.18be964c2b8feae5 namespace: kserve-ci-e2e-test resourceVersion: "14593" uid: ead525ee-d5c6-4ec1-83aa-d457015bc44a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 3a4bb54c-cae0-4723-ad7c-aca4f345044e kind: Event lastTimestamp: "2026-07-02T21:29:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:16Z" name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn.18be964c2ed2c925 namespace: kserve-ci-e2e-test resourceVersion: "14595" uid: 220391bd-a1cf-474f-bdad-4f00f08b2103 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 3a4bb54c-cae0-4723-ad7c-aca4f345044e kind: Event lastTimestamp: "2026-07-02T21:29:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:16Z" name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn.18be964c2f41ad03 namespace: kserve-ci-e2e-test resourceVersion: "14597" uid: f93e7541-99e7-44b2-8b14-bde7d9dc6426 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 3a4bb54c-cae0-4723-ad7c-aca4f345044e kind: Event lastTimestamp: "2026-07-02T21:29:16Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T21:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:16Z" name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn.18be964c2f4db86e namespace: kserve-ci-e2e-test resourceVersion: "14598" uid: cecf23c4-b9ad-4735-b8cd-8a1f01a2c1f3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 3a4bb54c-cae0-4723-ad7c-aca4f345044e kind: Event lastTimestamp: "2026-07-02T21:29:18Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.451s (2.451s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T21:29:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:18Z" name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn.18be964cc16e9986 namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: 18d11b8f-0313-4971-afa1-25a59777c070 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 3a4bb54c-cae0-4723-ad7c-aca4f345044e kind: Event lastTimestamp: "2026-07-02T21:29:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:29:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:18Z" name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn.18be964cc47281c9 namespace: kserve-ci-e2e-test resourceVersion: "14614" uid: 3d607ea3-ffdf-4015-96d3-a006b6bcd578 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 3a4bb54c-cae0-4723-ad7c-aca4f345044e kind: Event lastTimestamp: "2026-07-02T21:29:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:29:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:18Z" name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn.18be964cc4e61ec3 namespace: kserve-ci-e2e-test resourceVersion: "14615" uid: 87cad980-6a31-4217-9cb2-b43c67dcbe31 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 3a4bb54c-cae0-4723-ad7c-aca4f345044e kind: Event lastTimestamp: "2026-07-02T21:29:18Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:7fce5049d5e22f5c18756419ab95335e509bf71d40b49159f51a66f5ab33232f" metadata: creationTimestamp: "2026-07-02T21:29:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:18Z" name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn.18be964cc4f3cd9f namespace: kserve-ci-e2e-test resourceVersion: "14616" uid: 25d78b8f-841f-4d4e-a4c1-1b51582d8705 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 3a4bb54c-cae0-4723-ad7c-aca4f345044e kind: Event lastTimestamp: "2026-07-02T21:29:21Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:7fce5049d5e22f5c18756419ab95335e509bf71d40b49159f51a66f5ab33232f" in 2.403s (2.403s including waiting). Image size: 239997024 bytes.' metadata: creationTimestamp: "2026-07-02T21:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:21Z" name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn.18be964d543b4aa0 namespace: kserve-ci-e2e-test resourceVersion: "14631" uid: 572e38fa-be7b-4037-a7ac-404903f5a589 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 3a4bb54c-cae0-4723-ad7c-aca4f345044e kind: Event lastTimestamp: "2026-07-02T21:29:21Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-02T21:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:21Z" name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn.18be964d56ff4095 namespace: kserve-ci-e2e-test resourceVersion: "14632" uid: 874584fc-92d8-43fa-b809-6096cd3d3cc4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:29:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 3a4bb54c-cae0-4723-ad7c-aca4f345044e kind: Event lastTimestamp: "2026-07-02T21:29:21Z" message: Started container agent metadata: creationTimestamp: "2026-07-02T21:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:29:21Z" name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn.18be964d5771f323 namespace: kserve-ci-e2e-test resourceVersion: "14633" uid: 96959b5c-73f2-405b-822d-ed6978d162e1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T21:29:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 3a4bb54c-cae0-4723-ad7c-aca4f345044e kind: Event lastTimestamp: "2026-07-02T21:30:57Z" message: 'Readiness probe failed: dial tcp 10.132.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:57Z" name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn.18be964d617c0c76 namespace: kserve-ci-e2e-test resourceVersion: "15467" uid: b0317e67-478e-4533-ac8b-1b017ebac8ad reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T21:29:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 3a4bb54c-cae0-4723-ad7c-aca4f345044e kind: Event lastTimestamp: "2026-07-02T21:30:57Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T21:29:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:57Z" name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn.18be964d9d4e3cc3 namespace: kserve-ci-e2e-test resourceVersion: "15468" uid: 0c937d7f-670f-4fb9-b48a-fbd633be6a4e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 3a4bb54c-cae0-4723-ad7c-aca4f345044e kind: Event lastTimestamp: "2026-07-02T21:30:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:30:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:51Z" name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn.18be966265ea74b4 namespace: kserve-ci-e2e-test resourceVersion: "15342" uid: 1b061789-b3ac-4a26-8e71-a05200aa319c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 3a4bb54c-cae0-4723-ad7c-aca4f345044e kind: Event lastTimestamp: "2026-07-02T21:30:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:30:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:51Z" name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn.18be966265eaddfe namespace: kserve-ci-e2e-test resourceVersion: "15351" uid: 6be26b4e-64d9-4dcf-92c6-99c2a1898f70 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 3a4bb54c-cae0-4723-ad7c-aca4f345044e kind: Event lastTimestamp: "2026-07-02T21:30:51Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-02T21:30:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:51Z" name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn.18be966265eade17 namespace: kserve-ci-e2e-test resourceVersion: "15347" uid: 647c80d1-47d7-4752-8446-94e4bb18b9ca reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:30:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 3a4bb54c-cae0-4723-ad7c-aca4f345044e kind: Event lastTimestamp: "2026-07-02T21:30:58Z" message: 'Readiness probe failed: Get "https://10.132.0.23:8643/healthz": dial tcp 10.132.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:30:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:58Z" name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn.18be9662cd896f39 namespace: kserve-ci-e2e-test resourceVersion: "15478" uid: d945f284-4840-4c4f-b94d-428002dcb6ed reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:28:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9 namespace: kserve-ci-e2e-test resourceVersion: "14411" uid: 243a0a27-4bfb-4d9b-962b-3423e5ba580c kind: Event lastTimestamp: "2026-07-02T21:28:56Z" message: 'Created pod: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9-h4kfn' metadata: creationTimestamp: "2026-07-02T21:28:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:28:56Z" name: isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9.18be96479cab5ef6 namespace: kserve-ci-e2e-test resourceVersion: "14426" uid: 48c4455d-0eec-44e1-9dd3-44374f4a8de8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:28:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-ed912-predictor namespace: kserve-ci-e2e-test resourceVersion: "14410" uid: 97679aed-a848-4d09-b85c-5e732cdb0020 kind: Event lastTimestamp: "2026-07-02T21:28:56Z" message: Scaled up replica set isvc-raw-sklearn-batcher-ed912-predictor-98d8544d9 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:28:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:28:56Z" name: isvc-raw-sklearn-batcher-ed912-predictor.18be964793ab673d namespace: kserve-ci-e2e-test resourceVersion: "14412" uid: 3abf2c8a-1776-4171-ab9b-699773528337 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:29:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-ed912-predictor namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: 353b399a-4c69-443a-a6b1-89d99186ab33 kind: Event lastTimestamp: "2026-07-02T21:29:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:29:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:29:41Z" name: isvc-raw-sklearn-batcher-ed912-predictor.18be964b18f8e51e namespace: kserve-ci-e2e-test resourceVersion: "14784" uid: 5ffe70bf-2789-4e52-afde-2d24b5b0d2d1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:29:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-ed912-predictor namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: 353b399a-4c69-443a-a6b1-89d99186ab33 kind: Event lastTimestamp: "2026-07-02T21:29:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:29:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:29:41Z" name: isvc-raw-sklearn-batcher-ed912-predictor.18be964b194819a6 namespace: kserve-ci-e2e-test resourceVersion: "14785" uid: b988d686-c172-419a-af0e-03df86b05329 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:29:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-ed912-predictor namespace: kserve-ci-e2e-test resourceVersion: "14557" uid: 353b399a-4c69-443a-a6b1-89d99186ab33 kind: Event lastTimestamp: "2026-07-02T21:30:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:29:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:30:41Z" name: isvc-raw-sklearn-batcher-ed912-predictor.18be965595d302dd namespace: kserve-ci-e2e-test resourceVersion: "15232" uid: 7e6162ba-f949-48c5-92e9-62709a485b35 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:29:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-ed912-predictor namespace: kserve-ci-e2e-test resourceVersion: "14557" uid: 353b399a-4c69-443a-a6b1-89d99186ab33 kind: Event lastTimestamp: "2026-07-02T21:30:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:29:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:30:41Z" name: isvc-raw-sklearn-batcher-ed912-predictor.18be9655963bdf24 namespace: kserve-ci-e2e-test resourceVersion: "15233" uid: de3ccae1-7242-4180-a04b-7606dda4cd18 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T21:28:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-ed912 namespace: kserve-ci-e2e-test resourceVersion: "14401" uid: 93129318-3779-4203-86b0-6b4511c7d234 kind: Event lastTimestamp: "2026-07-02T21:30:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:28:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:30:37Z" name: isvc-raw-sklearn-batcher-ed912.18be964799c5ede0 namespace: kserve-ci-e2e-test resourceVersion: "15199" uid: c94bf59e-af55-4c2f-941a-3fbf4713edfc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:28:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-ed912 namespace: kserve-ci-e2e-test resourceVersion: "14401" uid: 93129318-3779-4203-86b0-6b4511c7d234 kind: Event lastTimestamp: "2026-07-02T21:28:56Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-ed912": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-ed912": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:28:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:28:56Z" name: isvc-raw-sklearn-batcher-ed912.18be96479a20f978 namespace: kserve-ci-e2e-test resourceVersion: "14437" uid: 015a2dd7-675d-4fad-9631-e89900f59734 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:28:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-ed912 namespace: kserve-ci-e2e-test resourceVersion: "14401" uid: 93129318-3779-4203-86b0-6b4511c7d234 kind: Event lastTimestamp: "2026-07-02T21:28:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-ed912": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:28:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:28:56Z" name: isvc-raw-sklearn-batcher-ed912.18be96479a21566e namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: 0cefe15f-d7cc-4b40-b951-7caf7f9dd1b3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-ed912 namespace: kserve-ci-e2e-test resourceVersion: "15196" uid: 93129318-3779-4203-86b0-6b4511c7d234 kind: Event lastTimestamp: "2026-07-02T21:30:37Z" message: InferenceService [isvc-raw-sklearn-batcher-ed912] is Ready metadata: creationTimestamp: "2026-07-02T21:30:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:30:37Z" name: isvc-raw-sklearn-batcher-ed912.18be965ef78bfeea namespace: kserve-ci-e2e-test resourceVersion: "15197" uid: f31a644c-84c3-4266-a14d-ca46eb9df33f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:40:33.265480Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-0bbdb6-predictor-7d847756c9-n48dr namespace: kserve-ci-e2e-test resourceVersion: "20340" uid: b62245da-2522-464c-95ef-70d7d24c0122 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-0bbdb6-predictor-7d847756c9-n48dr to ip-10-0-135-227.ec2.internal metadata: creationTimestamp: "2026-07-02T21:40:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:40:33Z" name: isvc-secondary-0bbdb6-predictor-7d847756c9-n48dr.18be96e9c535f833 namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: 65923bc2-7aff-438d-bddc-f49e9a7e5116 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-558f78ddf7-rgj7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:33Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-0bbdb6-predictor-7d847756c9-n48dr namespace: kserve-ci-e2e-test resourceVersion: "20350" uid: b62245da-2522-464c-95ef-70d7d24c0122 kind: Event lastTimestamp: "2026-07-02T21:40:33Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:40:33Z" name: isvc-secondary-0bbdb6-predictor-7d847756c9-n48dr.18be96e9de8af43d namespace: kserve-ci-e2e-test resourceVersion: "20359" uid: 1a959382-18e0-4027-800a-33befe2d05a2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:40:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-0bbdb6-predictor-7d847756c9-n48dr namespace: kserve-ci-e2e-test resourceVersion: "20342" uid: b62245da-2522-464c-95ef-70d7d24c0122 kind: Event lastTimestamp: "2026-07-02T21:40:38Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:33f82c067d520fe749f73749cf76b5935fe69bd9eb141504c6660698c7947743" already present on machine metadata: creationTimestamp: "2026-07-02T21:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:38Z" name: isvc-secondary-0bbdb6-predictor-7d847756c9-n48dr.18be96e9dfed8cc1 namespace: kserve-ci-e2e-test resourceVersion: "20404" uid: 26d79704-e430-47f8-902e-be31766ec24c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:40:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-0bbdb6-predictor-7d847756c9-n48dr namespace: kserve-ci-e2e-test resourceVersion: "20342" uid: b62245da-2522-464c-95ef-70d7d24c0122 kind: Event lastTimestamp: "2026-07-02T21:40:38Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T21:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:38Z" name: isvc-secondary-0bbdb6-predictor-7d847756c9-n48dr.18be96e9e372124a namespace: kserve-ci-e2e-test resourceVersion: "20408" uid: 26e4c3be-440e-40fd-83a8-0880ba9bf5cd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:40:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-0bbdb6-predictor-7d847756c9-n48dr namespace: kserve-ci-e2e-test resourceVersion: "20342" uid: b62245da-2522-464c-95ef-70d7d24c0122 kind: Event lastTimestamp: "2026-07-02T21:40:38Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T21:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:38Z" name: isvc-secondary-0bbdb6-predictor-7d847756c9-n48dr.18be96e9e3de514a namespace: kserve-ci-e2e-test resourceVersion: "20410" uid: e5a81985-bf99-4a34-a236-2b6f44e3f61c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-0bbdb6-predictor-7d847756c9-n48dr namespace: kserve-ci-e2e-test resourceVersion: "20342" uid: b62245da-2522-464c-95ef-70d7d24c0122 kind: Event lastTimestamp: "2026-07-02T21:40:42Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-0bbdb6-predictor-7d847756c9-n48dr_kserve-ci-e2e-test(b62245da-2522-464c-95ef-70d7d24c0122) metadata: creationTimestamp: "2026-07-02T21:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:40:42Z" name: isvc-secondary-0bbdb6-predictor-7d847756c9-n48dr.18be96ebe8bf96d1 namespace: kserve-ci-e2e-test resourceVersion: "20440" uid: 8a79df07-5a2e-4c50-8f80-d8a9c65599d8 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-0bbdb6-predictor-7d847756c9 namespace: kserve-ci-e2e-test resourceVersion: "20333" uid: ef51f3e9-1b12-4c40-8c51-2e2d5890ec8f kind: Event lastTimestamp: "2026-07-02T21:40:33Z" message: 'Created pod: isvc-secondary-0bbdb6-predictor-7d847756c9-n48dr' metadata: creationTimestamp: "2026-07-02T21:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:40:33Z" name: isvc-secondary-0bbdb6-predictor-7d847756c9.18be96e9c4c80783 namespace: kserve-ci-e2e-test resourceVersion: "20341" uid: cf022711-8b98-4f8f-820f-6ccb65765f12 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-0bbdb6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20331" uid: b9b0d4ce-ebc3-4204-aa10-3d5c2aec6887 kind: Event lastTimestamp: "2026-07-02T21:40:33Z" message: Scaled up replica set isvc-secondary-0bbdb6-predictor-7d847756c9 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:40:33Z" name: isvc-secondary-0bbdb6-predictor.18be96e9c1df0de9 namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: d2c4d61f-d94e-485f-84ca-5e24e9054559 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-0bbdb6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20332" uid: 8b6e7209-e75e-434d-b2f3-fc6b407c308b kind: Event lastTimestamp: "2026-07-02T21:40:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:40:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:40:48Z" name: isvc-secondary-0bbdb6-predictor.18be96ed40663fc0 namespace: kserve-ci-e2e-test resourceVersion: "20488" uid: 384a948a-5aba-49a3-8a13-f5b1e8703702 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-0bbdb6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20332" uid: 8b6e7209-e75e-434d-b2f3-fc6b407c308b kind: Event lastTimestamp: "2026-07-02T21:40:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:40:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:40:48Z" name: isvc-secondary-0bbdb6-predictor.18be96ed40cc4651 namespace: kserve-ci-e2e-test resourceVersion: "20490" uid: e6ccbb93-98ac-4a5c-bf1d-f9ab1923cc86 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T21:40:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-0bbdb6 namespace: kserve-ci-e2e-test resourceVersion: "20324" uid: e058945e-17a4-4d7d-8ce9-6a367c66ac90 kind: Event lastTimestamp: "2026-07-02T21:40:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:40:42Z" name: isvc-secondary-0bbdb6.18be96e9c1f2e975 namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: 88f545c6-5241-4ef7-9e81-5a4bc606064c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-0bbdb6 namespace: kserve-ci-e2e-test resourceVersion: "20324" uid: e058945e-17a4-4d7d-8ce9-6a367c66ac90 kind: Event lastTimestamp: "2026-07-02T21:40:33Z" message: 'Failed to update status for InferenceService "isvc-secondary-0bbdb6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-0bbdb6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:40:33Z" name: isvc-secondary-0bbdb6.18be96e9c259da42 namespace: kserve-ci-e2e-test resourceVersion: "20337" uid: 91d3686c-3bc6-41a7-9cf4-49df62b0f8fb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:40:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-0bbdb6 namespace: kserve-ci-e2e-test resourceVersion: "20324" uid: e058945e-17a4-4d7d-8ce9-6a367c66ac90 kind: Event lastTimestamp: "2026-07-02T21:40:33Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-0bbdb6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:40:33Z" name: isvc-secondary-0bbdb6.18be96e9c25a2c29 namespace: kserve-ci-e2e-test resourceVersion: "20339" uid: fa5d5537-ebf2-4204-a3dc-cedd36a5bd91 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:30:51.859648Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn namespace: kserve-ci-e2e-test resourceVersion: "15339" uid: fa460ab6-5679-4644-8562-424253a015c3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn to ip-10-0-135-227.ec2.internal metadata: creationTimestamp: "2026-07-02T21:30:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:30:51Z" name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn.18be966266b8f728 namespace: kserve-ci-e2e-test resourceVersion: "15352" uid: 85ffd918-c014-4c1e-ae10-1f1454a7b75b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-558f78ddf7-rgj7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn namespace: kserve-ci-e2e-test resourceVersion: "15362" uid: fa460ab6-5679-4644-8562-424253a015c3 kind: Event lastTimestamp: "2026-07-02T21:30:52Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:30:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:30:52Z" name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn.18be96628051681a namespace: kserve-ci-e2e-test resourceVersion: "15394" uid: 208b27e7-6bff-489b-942d-7c72a596d283 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: fa460ab6-5679-4644-8562-424253a015c3 kind: Event lastTimestamp: "2026-07-02T21:30:52Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:33f82c067d520fe749f73749cf76b5935fe69bd9eb141504c6660698c7947743" already present on machine metadata: creationTimestamp: "2026-07-02T21:30:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:52Z" name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn.18be966281b0416d namespace: kserve-ci-e2e-test resourceVersion: "15396" uid: 2cd375e6-f0a0-4b6c-b337-faed05f58aa9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: fa460ab6-5679-4644-8562-424253a015c3 kind: Event lastTimestamp: "2026-07-02T21:30:52Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T21:30:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:52Z" name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn.18be966284d4dbf4 namespace: kserve-ci-e2e-test resourceVersion: "15398" uid: 909bb372-76ce-439d-b211-f39c7d6dd015 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: fa460ab6-5679-4644-8562-424253a015c3 kind: Event lastTimestamp: "2026-07-02T21:30:52Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T21:30:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:52Z" name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn.18be9662855c8aae namespace: kserve-ci-e2e-test resourceVersion: "15399" uid: fee1769c-3ece-456d-be31-6a84fb8f89cb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: fa460ab6-5679-4644-8562-424253a015c3 kind: Event lastTimestamp: "2026-07-02T21:30:55Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:55Z" name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn.18be96635b74c745 namespace: kserve-ci-e2e-test resourceVersion: "15443" uid: f54aa39f-02fb-4bc2-9393-5a0d7d6ac333 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: fa460ab6-5679-4644-8562-424253a015c3 kind: Event lastTimestamp: "2026-07-02T21:30:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:56Z" name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn.18be96636234e21f namespace: kserve-ci-e2e-test resourceVersion: "15451" uid: 68949f05-a265-493b-8fbe-896693448d08 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: fa460ab6-5679-4644-8562-424253a015c3 kind: Event lastTimestamp: "2026-07-02T21:30:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:56Z" name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn.18be9663628fbe0f namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: 1eb0dabe-f1e4-401c-800e-b6791be49414 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: fa460ab6-5679-4644-8562-424253a015c3 kind: Event lastTimestamp: "2026-07-02T21:30:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:56Z" name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn.18be9663629905a0 namespace: kserve-ci-e2e-test resourceVersion: "15453" uid: 7bc1e377-d2fe-4aa9-9304-0748c14f9732 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: fa460ab6-5679-4644-8562-424253a015c3 kind: Event lastTimestamp: "2026-07-02T21:30:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:56Z" name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn.18be966364f4b1b7 namespace: kserve-ci-e2e-test resourceVersion: "15454" uid: 735a0b0f-454a-4e63-8a19-a516ea95442a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: fa460ab6-5679-4644-8562-424253a015c3 kind: Event lastTimestamp: "2026-07-02T21:30:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:56Z" name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn.18be966365501f0a namespace: kserve-ci-e2e-test resourceVersion: "15456" uid: 16cd90c5-de3b-4c58-8912-716066afe192 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T21:30:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: fa460ab6-5679-4644-8562-424253a015c3 kind: Event lastTimestamp: "2026-07-02T21:32:46Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:46Z" name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn.18be966397ba2eb4 namespace: kserve-ci-e2e-test resourceVersion: "16478" uid: 87480409-6714-4a5c-973f-6fb5ce65a7c5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: fa460ab6-5679-4644-8562-424253a015c3 kind: Event lastTimestamp: "2026-07-02T21:32:42Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:42Z" name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn.18be967c19bf8707 namespace: kserve-ci-e2e-test resourceVersion: "16370" uid: d02f130f-56ce-4972-a4b4-9edd7d549031 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: fa460ab6-5679-4644-8562-424253a015c3 kind: Event lastTimestamp: "2026-07-02T21:32:42Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:42Z" name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn.18be967c19c0ba77 namespace: kserve-ci-e2e-test resourceVersion: "16374" uid: 2fd5e65b-97b4-4c14-8840-45d16c796b8a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: fa460ab6-5679-4644-8562-424253a015c3 kind: Event lastTimestamp: "2026-07-02T21:32:42Z" message: 'Readiness probe failed: Get "https://10.132.0.24:8643/healthz": dial tcp 10.132.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:42Z" name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn.18be967c46587afb namespace: kserve-ci-e2e-test resourceVersion: "16431" uid: 05339225-d2e9-485c-9f92-480823648125 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445 namespace: kserve-ci-e2e-test resourceVersion: "15313" uid: c37dd094-1e14-4104-8b4d-9cbaed3b894a kind: Event lastTimestamp: "2026-07-02T21:30:51Z" message: 'Created pod: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445-v8mhn' metadata: creationTimestamp: "2026-07-02T21:30:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:30:51Z" name: isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445.18be96626623ddea namespace: kserve-ci-e2e-test resourceVersion: "15343" uid: 2bb8e83c-264d-4b92-8062-e98dbf189280 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-c1b1e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15311" uid: 656dca00-915a-4a44-8b54-9d667f78c0d7 kind: Event lastTimestamp: "2026-07-02T21:30:51Z" message: Scaled up replica set isvc-sklearn-graph-raw-c1b1e-predictor-56f778f445 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:30:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:30:51Z" name: isvc-sklearn-graph-raw-c1b1e-predictor.18be9662609de026 namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: ab553ca8-10f5-4939-800b-f72bff34f308 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-c1b1e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15312" uid: 1f75c76a-5e61-493c-9e7d-1504ef45b38f kind: Event lastTimestamp: "2026-07-02T21:31:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:31:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:06Z" name: isvc-sklearn-graph-raw-c1b1e-predictor.18be9665dfa6e5de namespace: kserve-ci-e2e-test resourceVersion: "15537" uid: d7273858-115a-4eb4-b42d-6e2be490e65b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-c1b1e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15312" uid: 1f75c76a-5e61-493c-9e7d-1504ef45b38f kind: Event lastTimestamp: "2026-07-02T21:31:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:31:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:06Z" name: isvc-sklearn-graph-raw-c1b1e-predictor.18be9665e003a4e5 namespace: kserve-ci-e2e-test resourceVersion: "15539" uid: 1ca0c67f-41a3-4338-90e7-87d877a5c3a9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:31:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-c1b1e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: 1f75c76a-5e61-493c-9e7d-1504ef45b38f kind: Event lastTimestamp: "2026-07-02T21:32:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:31:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:36Z" name: isvc-sklearn-graph-raw-c1b1e-predictor.18be96695ebec10b namespace: kserve-ci-e2e-test resourceVersion: "16259" uid: a2367f32-6f16-4eb9-882f-db2a8c318a86 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:31:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-c1b1e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: 1f75c76a-5e61-493c-9e7d-1504ef45b38f kind: Event lastTimestamp: "2026-07-02T21:32:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:31:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:36Z" name: isvc-sklearn-graph-raw-c1b1e-predictor.18be96695f2920e5 namespace: kserve-ci-e2e-test resourceVersion: "16260" uid: 75fdbbf7-0a0d-427e-9b07-ec751a9b413c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T21:30:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-c1b1e namespace: kserve-ci-e2e-test resourceVersion: "15301" uid: 7208b29f-3082-4c66-8bd8-dd5fa7bf2a3d kind: Event lastTimestamp: "2026-07-02T21:32:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:30:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:32:16Z" name: isvc-sklearn-graph-raw-c1b1e.18be966260a35281 namespace: kserve-ci-e2e-test resourceVersion: "16070" uid: 29705f7f-c7bf-452b-a563-c45e82c9357d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-c1b1e namespace: kserve-ci-e2e-test resourceVersion: "15301" uid: 7208b29f-3082-4c66-8bd8-dd5fa7bf2a3d kind: Event lastTimestamp: "2026-07-02T21:30:51Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-c1b1e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-c1b1e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:30:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:30:51Z" name: isvc-sklearn-graph-raw-c1b1e.18be96626137e8d8 namespace: kserve-ci-e2e-test resourceVersion: "15317" uid: 1d3b31db-27f2-4cfb-b92d-b11cd7c46f7a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-c1b1e namespace: kserve-ci-e2e-test resourceVersion: "16067" uid: 7208b29f-3082-4c66-8bd8-dd5fa7bf2a3d kind: Event lastTimestamp: "2026-07-02T21:32:16Z" message: InferenceService [isvc-sklearn-graph-raw-c1b1e] is Ready metadata: creationTimestamp: "2026-07-02T21:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:32:16Z" name: isvc-sklearn-graph-raw-c1b1e.18be967605e71f7a namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: db3d8dfc-f306-4495-806c-59ca467cbe07 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:32:42.218381Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd namespace: kserve-ci-e2e-test resourceVersion: "16352" uid: 7273526a-482b-44bd-801d-dc5ce04f40f7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd to ip-10-0-135-227.ec2.internal metadata: creationTimestamp: "2026-07-02T21:32:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:32:42Z" name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd.18be967c189da6f6 namespace: kserve-ci-e2e-test resourceVersion: "16361" uid: b93929cb-70ad-40d5-a2d6-6af570ecf476 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-558f78ddf7-rgj7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:42Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: 7273526a-482b-44bd-801d-dc5ce04f40f7 kind: Event lastTimestamp: "2026-07-02T21:32:42Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-raw-hpa-af6d2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T21:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:42Z" name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd.18be967c220f741a namespace: kserve-ci-e2e-test resourceVersion: "16415" uid: ffc3e408-a1e3-477b-8410-5af40cab8c18 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:43Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd namespace: kserve-ci-e2e-test resourceVersion: "16375" uid: 7273526a-482b-44bd-801d-dc5ce04f40f7 kind: Event lastTimestamp: "2026-07-02T21:32:43Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:32:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:32:43Z" name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd.18be967c5607e5ee namespace: kserve-ci-e2e-test resourceVersion: "16438" uid: 305a2239-8dd5-4155-b43f-18dcc223216b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: 7273526a-482b-44bd-801d-dc5ce04f40f7 kind: Event lastTimestamp: "2026-07-02T21:32:43Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:33f82c067d520fe749f73749cf76b5935fe69bd9eb141504c6660698c7947743" already present on machine metadata: creationTimestamp: "2026-07-02T21:32:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:43Z" name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd.18be967c5787c237 namespace: kserve-ci-e2e-test resourceVersion: "16440" uid: 347ed74a-c2d9-4c59-97e9-b74cebaeb74c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: 7273526a-482b-44bd-801d-dc5ce04f40f7 kind: Event lastTimestamp: "2026-07-02T21:32:43Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T21:32:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:43Z" name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd.18be967c5b68b254 namespace: kserve-ci-e2e-test resourceVersion: "16442" uid: 3a795250-a8c9-492f-96d9-a6f10cc147ea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: 7273526a-482b-44bd-801d-dc5ce04f40f7 kind: Event lastTimestamp: "2026-07-02T21:32:43Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T21:32:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:43Z" name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd.18be967c5bca46a0 namespace: kserve-ci-e2e-test resourceVersion: "16443" uid: f6ec96ee-e790-4de3-b692-a1402544c3d2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: 7273526a-482b-44bd-801d-dc5ce04f40f7 kind: Event lastTimestamp: "2026-07-02T21:32:47Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:47Z" name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd.18be967d519fc627 namespace: kserve-ci-e2e-test resourceVersion: "16492" uid: 07ae2691-6301-47ca-9250-d6b41ef9aa83 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: 7273526a-482b-44bd-801d-dc5ce04f40f7 kind: Event lastTimestamp: "2026-07-02T21:32:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:47Z" name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd.18be967d556dd4e5 namespace: kserve-ci-e2e-test resourceVersion: "16500" uid: 7f88e269-4e3a-47ac-908c-e1af8a993096 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: 7273526a-482b-44bd-801d-dc5ce04f40f7 kind: Event lastTimestamp: "2026-07-02T21:32:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:47Z" name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd.18be967d55f9eb0d namespace: kserve-ci-e2e-test resourceVersion: "16504" uid: 2e92ecec-ebc4-4812-8c68-b90802ebd6c3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: 7273526a-482b-44bd-801d-dc5ce04f40f7 kind: Event lastTimestamp: "2026-07-02T21:32:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:47Z" name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd.18be967d56048c8d namespace: kserve-ci-e2e-test resourceVersion: "16506" uid: 77786917-2557-4c36-92e5-c5c5f4bd4486 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: 7273526a-482b-44bd-801d-dc5ce04f40f7 kind: Event lastTimestamp: "2026-07-02T21:32:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:47Z" name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd.18be967d58c429ae namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: 6ab6448b-cf5f-4a7d-af57-3b38d5f35f72 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: 7273526a-482b-44bd-801d-dc5ce04f40f7 kind: Event lastTimestamp: "2026-07-02T21:32:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:47Z" name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd.18be967d5940539f namespace: kserve-ci-e2e-test resourceVersion: "16512" uid: c5f39210-4a41-4510-9b9a-a2413dcf15a0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-02T21:32:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: 7273526a-482b-44bd-801d-dc5ce04f40f7 kind: Event lastTimestamp: "2026-07-02T21:34:36Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:32:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:36Z" name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd.18be967d8e13278c namespace: kserve-ci-e2e-test resourceVersion: "17471" uid: bd9feed9-f453-4337-9bdc-6c2189612a61 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: 7273526a-482b-44bd-801d-dc5ce04f40f7 kind: Event lastTimestamp: "2026-07-02T21:34:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:34:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:32Z" name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd.18be9695c919a5ad namespace: kserve-ci-e2e-test resourceVersion: "17404" uid: 6247a4e6-fd68-44ac-8c05-45115b0b27b1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: 7273526a-482b-44bd-801d-dc5ce04f40f7 kind: Event lastTimestamp: "2026-07-02T21:34:32Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:34:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:32Z" name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd.18be9695c91acbd5 namespace: kserve-ci-e2e-test resourceVersion: "17408" uid: 9c0b39ff-45e8-41ad-8c7c-8cdd2378103c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: 7273526a-482b-44bd-801d-dc5ce04f40f7 kind: Event lastTimestamp: "2026-07-02T21:34:34Z" message: 'Readiness probe failed: Get "https://10.132.0.26:8643/healthz": dial tcp 10.132.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:34:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:34Z" name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd.18be96963c600aa6 namespace: kserve-ci-e2e-test resourceVersion: "17448" uid: 5fd481a3-1131-4f67-905f-6fb17e47191e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697 namespace: kserve-ci-e2e-test resourceVersion: "16331" uid: 8336f407-661a-49b5-b7d8-f68367f10536 kind: Event lastTimestamp: "2026-07-02T21:32:42Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697-mbgrd' metadata: creationTimestamp: "2026-07-02T21:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:42Z" name: isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697.18be967c1832fb4b namespace: kserve-ci-e2e-test resourceVersion: "16356" uid: c906ee3a-ca06-4097-93c2-0a76e572f679 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-af6d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16328" uid: eb3b588b-f011-43ad-be7a-0782de5e5ced kind: Event lastTimestamp: "2026-07-02T21:32:42Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-af6d2-predictor-7bd4dcc697 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:42Z" name: isvc-sklearn-graph-raw-hpa-af6d2-predictor.18be967c12ed4c27 namespace: kserve-ci-e2e-test resourceVersion: "16332" uid: 77ae5f3b-3301-48a3-98cc-4e7a1eb3f793 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:32:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-af6d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: facd0d7a-afde-4edb-bc7d-c01ca3f6d3ec kind: Event lastTimestamp: "2026-07-02T21:33:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:32:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:33:12Z" name: isvc-sklearn-graph-raw-hpa-af6d2-predictor.18be967f91825062 namespace: kserve-ci-e2e-test resourceVersion: "16705" uid: cdd6a60f-b26a-42ef-9aa0-d52bca9de8bb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:32:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-af6d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: facd0d7a-afde-4edb-bc7d-c01ca3f6d3ec kind: Event lastTimestamp: "2026-07-02T21:33:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:32:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:33:12Z" name: isvc-sklearn-graph-raw-hpa-af6d2-predictor.18be967f91e2beaf namespace: kserve-ci-e2e-test resourceVersion: "16706" uid: e2d87bcb-99b5-4741-b973-efedca920ffa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:33:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-af6d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16594" uid: facd0d7a-afde-4edb-bc7d-c01ca3f6d3ec kind: Event lastTimestamp: "2026-07-02T21:34:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:33:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:34:12Z" name: isvc-sklearn-graph-raw-hpa-af6d2-predictor.18be96868f856fdf namespace: kserve-ci-e2e-test resourceVersion: "17155" uid: 3b3ca0fa-e026-4fe9-889a-7d9974372b41 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:33:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-af6d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16594" uid: facd0d7a-afde-4edb-bc7d-c01ca3f6d3ec kind: Event lastTimestamp: "2026-07-02T21:34:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:33:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:34:12Z" name: isvc-sklearn-graph-raw-hpa-af6d2-predictor.18be96868fe7cf67 namespace: kserve-ci-e2e-test resourceVersion: "17156" uid: 79279fa9-cf1c-43ab-a562-d27ddbbd2a0f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T21:32:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-af6d2 namespace: kserve-ci-e2e-test resourceVersion: "16319" uid: c6bf4ac6-4063-4bd0-bb25-66beb5aa091b kind: Event lastTimestamp: "2026-07-02T21:34:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:34:06Z" name: isvc-sklearn-graph-raw-hpa-af6d2.18be967c12f13dcc namespace: kserve-ci-e2e-test resourceVersion: "17114" uid: c166a3fa-b70f-4a68-9194-9111c3e02380 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-af6d2 namespace: kserve-ci-e2e-test resourceVersion: "16319" uid: c6bf4ac6-4063-4bd0-bb25-66beb5aa091b kind: Event lastTimestamp: "2026-07-02T21:32:42Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-af6d2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-af6d2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:32:42Z" name: isvc-sklearn-graph-raw-hpa-af6d2.18be967c1397bfdf namespace: kserve-ci-e2e-test resourceVersion: "16335" uid: 96f41895-7168-4b64-890c-fb810f27b0e5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-af6d2 namespace: kserve-ci-e2e-test resourceVersion: "17112" uid: c6bf4ac6-4063-4bd0-bb25-66beb5aa091b kind: Event lastTimestamp: "2026-07-02T21:34:06Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-af6d2] is Ready metadata: creationTimestamp: "2026-07-02T21:34:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:34:06Z" name: isvc-sklearn-graph-raw-hpa-af6d2.18be968fa02a8dc8 namespace: kserve-ci-e2e-test resourceVersion: "17113" uid: 2e6500e9-39f9-4fa6-a8a1-29150d9741ea reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:36:27.804077Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb namespace: kserve-ci-e2e-test resourceVersion: "18409" uid: a693b634-4747-43d7-8648-ff8d2843ad6d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb to ip-10-0-135-227.ec2.internal metadata: creationTimestamp: "2026-07-02T21:36:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:36:27Z" name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb.18be96b09e9238bd namespace: kserve-ci-e2e-test resourceVersion: "18419" uid: df0b9f5f-5f67-40c0-a7bb-117541f9b629 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-558f78ddf7-rgj7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:36:27Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb namespace: kserve-ci-e2e-test resourceVersion: "18414" uid: a693b634-4747-43d7-8648-ff8d2843ad6d kind: Event lastTimestamp: "2026-07-02T21:36:27Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-scale-raw-e3b87-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T21:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:36:27Z" name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb.18be96b0a8c1773d namespace: kserve-ci-e2e-test resourceVersion: "18454" uid: 9b34afd6-71e8-4314-a9b6-401ac325cba9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:36:28Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb namespace: kserve-ci-e2e-test resourceVersion: "18426" uid: a693b634-4747-43d7-8648-ff8d2843ad6d kind: Event lastTimestamp: "2026-07-02T21:36:28Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:36:28Z" name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb.18be96b0dbeadad8 namespace: kserve-ci-e2e-test resourceVersion: "18465" uid: fb9f9766-484c-4a92-b327-5b8e3a903705 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb namespace: kserve-ci-e2e-test resourceVersion: "18414" uid: a693b634-4747-43d7-8648-ff8d2843ad6d kind: Event lastTimestamp: "2026-07-02T21:36:28Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:33f82c067d520fe749f73749cf76b5935fe69bd9eb141504c6660698c7947743" already present on machine metadata: creationTimestamp: "2026-07-02T21:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:36:28Z" name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb.18be96b0dd4cc468 namespace: kserve-ci-e2e-test resourceVersion: "18468" uid: 17a98fc5-36b6-4712-a9e5-0d2711ddc09b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb namespace: kserve-ci-e2e-test resourceVersion: "18414" uid: a693b634-4747-43d7-8648-ff8d2843ad6d kind: Event lastTimestamp: "2026-07-02T21:36:28Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T21:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:36:28Z" name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb.18be96b0e095ddef namespace: kserve-ci-e2e-test resourceVersion: "18470" uid: de12ab13-cf84-4da5-b11e-b52c77b8d8af reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb namespace: kserve-ci-e2e-test resourceVersion: "18414" uid: a693b634-4747-43d7-8648-ff8d2843ad6d kind: Event lastTimestamp: "2026-07-02T21:36:28Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T21:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:36:28Z" name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb.18be96b0e103ef9f namespace: kserve-ci-e2e-test resourceVersion: "18471" uid: 6b6c4372-d497-40e0-9007-ce93f81cc01d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:36:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb namespace: kserve-ci-e2e-test resourceVersion: "18414" uid: a693b634-4747-43d7-8648-ff8d2843ad6d kind: Event lastTimestamp: "2026-07-02T21:36:33Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:36:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:36:33Z" name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb.18be96b1ee1466dd namespace: kserve-ci-e2e-test resourceVersion: "18511" uid: dc7aeea9-d98c-49d1-81c7-1c8634e9c8bb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:36:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb namespace: kserve-ci-e2e-test resourceVersion: "18414" uid: a693b634-4747-43d7-8648-ff8d2843ad6d kind: Event lastTimestamp: "2026-07-02T21:36:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:36:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:36:33Z" name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb.18be96b1f112c795 namespace: kserve-ci-e2e-test resourceVersion: "18513" uid: 1cc52537-7a37-4716-8ba1-cbffbf30fbff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:36:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb namespace: kserve-ci-e2e-test resourceVersion: "18414" uid: a693b634-4747-43d7-8648-ff8d2843ad6d kind: Event lastTimestamp: "2026-07-02T21:36:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:36:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:36:33Z" name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb.18be96b1f18985b2 namespace: kserve-ci-e2e-test resourceVersion: "18514" uid: b9962980-1303-48f3-87fe-4bd35a305048 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:36:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb namespace: kserve-ci-e2e-test resourceVersion: "18414" uid: a693b634-4747-43d7-8648-ff8d2843ad6d kind: Event lastTimestamp: "2026-07-02T21:36:33Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:36:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:36:33Z" name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb.18be96b1f1939c3b namespace: kserve-ci-e2e-test resourceVersion: "18517" uid: 5654d710-7de2-432d-9d78-cc9c9f198959 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:36:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb namespace: kserve-ci-e2e-test resourceVersion: "18414" uid: a693b634-4747-43d7-8648-ff8d2843ad6d kind: Event lastTimestamp: "2026-07-02T21:36:33Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:36:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:36:33Z" name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb.18be96b1f434c6f4 namespace: kserve-ci-e2e-test resourceVersion: "18519" uid: e7a92d02-9719-4030-82fc-b1b23d208714 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:36:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb namespace: kserve-ci-e2e-test resourceVersion: "18414" uid: a693b634-4747-43d7-8648-ff8d2843ad6d kind: Event lastTimestamp: "2026-07-02T21:36:33Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:36:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:36:33Z" name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb.18be96b1f4a15005 namespace: kserve-ci-e2e-test resourceVersion: "18520" uid: 3bd37155-96ac-4a7b-a129-04c727a95735 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 19 eventTime: null firstTimestamp: "2026-07-02T21:36:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb namespace: kserve-ci-e2e-test resourceVersion: "18414" uid: a693b634-4747-43d7-8648-ff8d2843ad6d kind: Event lastTimestamp: "2026-07-02T21:39:12Z" message: 'Readiness probe failed: dial tcp 10.132.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:36:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:39:12Z" name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb.18be96b22a6ba036 namespace: kserve-ci-e2e-test resourceVersion: "19672" uid: c5fd4477-858a-40b9-a83e-11b9c829577f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:39:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb namespace: kserve-ci-e2e-test resourceVersion: "18414" uid: a693b634-4747-43d7-8648-ff8d2843ad6d kind: Event lastTimestamp: "2026-07-02T21:39:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:39:08Z" name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb.18be96d5ebd3d904 namespace: kserve-ci-e2e-test resourceVersion: "19643" uid: 4daee6f3-73d3-408d-83dd-0f6152a11562 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:39:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb namespace: kserve-ci-e2e-test resourceVersion: "18414" uid: a693b634-4747-43d7-8648-ff8d2843ad6d kind: Event lastTimestamp: "2026-07-02T21:39:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:39:08Z" name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb.18be96d5ebd48377 namespace: kserve-ci-e2e-test resourceVersion: "19644" uid: d9b944ec-fb3d-4907-86a8-6d5516016d16 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:39:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb namespace: kserve-ci-e2e-test resourceVersion: "18414" uid: a693b634-4747-43d7-8648-ff8d2843ad6d kind: Event lastTimestamp: "2026-07-02T21:39:16Z" message: 'Readiness probe failed: Get "https://10.132.0.30:8643/healthz": dial tcp 10.132.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:39:16Z" name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb.18be96d6b8bd3f15 namespace: kserve-ci-e2e-test resourceVersion: "19751" uid: 8fa9ef88-b0fb-47fc-9333-7f7be3454141 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:36:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc namespace: kserve-ci-e2e-test resourceVersion: "18392" uid: 6425a82f-845e-4875-8bf8-400c01a95ded kind: Event lastTimestamp: "2026-07-02T21:36:27Z" message: 'Created pod: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc-62kdb' metadata: creationTimestamp: "2026-07-02T21:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:36:27Z" name: isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc.18be96b09e26fc10 namespace: kserve-ci-e2e-test resourceVersion: "18413" uid: eefb3cf5-8344-4416-a611-cc91d63d1142 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:36:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-e3b87-predictor namespace: kserve-ci-e2e-test resourceVersion: "18390" uid: 5b517f45-5baa-40d6-b1a9-8a39ceade5c0 kind: Event lastTimestamp: "2026-07-02T21:36:27Z" message: Scaled up replica set isvc-sklearn-scale-raw-e3b87-predictor-6c9ddc44bc from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:36:27Z" name: isvc-sklearn-scale-raw-e3b87-predictor.18be96b099a61f75 namespace: kserve-ci-e2e-test resourceVersion: "18393" uid: a2284008-7566-4538-8d7f-f0406cf7fe83 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:36:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-e3b87-predictor namespace: kserve-ci-e2e-test resourceVersion: "18391" uid: 6a2e337c-e6d1-4c41-980a-319100786ec4 kind: Event lastTimestamp: "2026-07-02T21:36:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:36:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:36:57Z" name: isvc-sklearn-scale-raw-e3b87-predictor.18be96b41892004a namespace: kserve-ci-e2e-test resourceVersion: "18701" uid: ca2e7d73-bea8-42c8-bec4-cc8b92e34171 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:36:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-e3b87-predictor namespace: kserve-ci-e2e-test resourceVersion: "18391" uid: 6a2e337c-e6d1-4c41-980a-319100786ec4 kind: Event lastTimestamp: "2026-07-02T21:36:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:36:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:36:57Z" name: isvc-sklearn-scale-raw-e3b87-predictor.18be96b418ee4bef namespace: kserve-ci-e2e-test resourceVersion: "18702" uid: 4baf2e62-e70d-4b2e-ac3e-d96d118d9731 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-02T21:37:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-e3b87-predictor namespace: kserve-ci-e2e-test resourceVersion: "18593" uid: 6a2e337c-e6d1-4c41-980a-319100786ec4 kind: Event lastTimestamp: "2026-07-02T21:38:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:37:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:38:57Z" name: isvc-sklearn-scale-raw-e3b87-predictor.18be96bb16f19f5b namespace: kserve-ci-e2e-test resourceVersion: "19547" uid: 7082e293-41d0-493d-aecd-5c27689b9d1d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-02T21:37:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-e3b87-predictor namespace: kserve-ci-e2e-test resourceVersion: "18593" uid: 6a2e337c-e6d1-4c41-980a-319100786ec4 kind: Event lastTimestamp: "2026-07-02T21:38:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:37:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:38:57Z" name: isvc-sklearn-scale-raw-e3b87-predictor.18be96bb1758d585 namespace: kserve-ci-e2e-test resourceVersion: "19548" uid: 8d309d68-c4f6-4c06-ba57-464c4df42bbd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T21:36:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-e3b87 namespace: kserve-ci-e2e-test resourceVersion: "18381" uid: 2dd0480b-9889-4732-8014-e0ab5f016b2b kind: Event lastTimestamp: "2026-07-02T21:39:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:39:02Z" name: isvc-sklearn-scale-raw-e3b87.18be96b099ae12df namespace: kserve-ci-e2e-test resourceVersion: "19584" uid: 37d89f69-62e2-484e-8d29-299a8111e2f4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:36:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-e3b87 namespace: kserve-ci-e2e-test resourceVersion: "18381" uid: 2dd0480b-9889-4732-8014-e0ab5f016b2b kind: Event lastTimestamp: "2026-07-02T21:36:27Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-e3b87": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-e3b87": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:36:27Z" name: isvc-sklearn-scale-raw-e3b87.18be96b09a1cec55 namespace: kserve-ci-e2e-test resourceVersion: "18396" uid: dda37f0b-11d4-42ed-a072-373ed8f94d9a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:39:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-e3b87 namespace: kserve-ci-e2e-test resourceVersion: "19582" uid: 2dd0480b-9889-4732-8014-e0ab5f016b2b kind: Event lastTimestamp: "2026-07-02T21:39:02Z" message: InferenceService [isvc-sklearn-scale-raw-e3b87] is Ready metadata: creationTimestamp: "2026-07-02T21:39:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:39:02Z" name: isvc-sklearn-scale-raw-e3b87.18be96d48c761b85 namespace: kserve-ci-e2e-test resourceVersion: "19583" uid: aac392c3-15ba-4f8a-a784-7e7c667031bb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:30:51.949374Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw namespace: kserve-ci-e2e-test resourceVersion: "15370" uid: f252efa2-359f-4eb0-8a51-c2cf43f952af kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw to ip-10-0-135-227.ec2.internal metadata: creationTimestamp: "2026-07-02T21:30:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:30:51Z" name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw.18be96626c12243d namespace: kserve-ci-e2e-test resourceVersion: "15376" uid: ddb02df6-6c93-4369-9db4-57f49a10a8d0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-558f78ddf7-rgj7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw namespace: kserve-ci-e2e-test resourceVersion: "15382" uid: f252efa2-359f-4eb0-8a51-c2cf43f952af kind: Event lastTimestamp: "2026-07-02T21:30:52Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:30:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:30:52Z" name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw.18be966285d65805 namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: 73cb17f1-99d3-4b15-a851-dfe376d59b6f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: f252efa2-359f-4eb0-8a51-c2cf43f952af kind: Event lastTimestamp: "2026-07-02T21:30:52Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:33f82c067d520fe749f73749cf76b5935fe69bd9eb141504c6660698c7947743" already present on machine metadata: creationTimestamp: "2026-07-02T21:30:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:52Z" name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw.18be9662874223e5 namespace: kserve-ci-e2e-test resourceVersion: "15402" uid: 8c5addf3-6939-4713-a4e6-86912744be80 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: f252efa2-359f-4eb0-8a51-c2cf43f952af kind: Event lastTimestamp: "2026-07-02T21:30:52Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T21:30:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:52Z" name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw.18be96628a47d4c1 namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 744aa3d9-3567-409d-8910-1512783f3993 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: f252efa2-359f-4eb0-8a51-c2cf43f952af kind: Event lastTimestamp: "2026-07-02T21:30:52Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T21:30:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:52Z" name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw.18be96628aab8ce6 namespace: kserve-ci-e2e-test resourceVersion: "15404" uid: 4f969056-ee52-4786-9744-8a43749fd7fe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: f252efa2-359f-4eb0-8a51-c2cf43f952af kind: Event lastTimestamp: "2026-07-02T21:30:55Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-02T21:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:30:55Z" name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw.18be96635b230490 namespace: kserve-ci-e2e-test resourceVersion: "15442" uid: 7e54e369-d8ae-4617-ab92-995660eaa436 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: f252efa2-359f-4eb0-8a51-c2cf43f952af kind: Event lastTimestamp: "2026-07-02T21:31:15Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 19.77s (19.77s including waiting). Image size: 1305293048 bytes.' metadata: creationTimestamp: "2026-07-02T21:31:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:15Z" name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw.18be9667f5914a0a namespace: kserve-ci-e2e-test resourceVersion: "15604" uid: 98694365-b4f1-402c-98d5-ddeba22ec1dd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: f252efa2-359f-4eb0-8a51-c2cf43f952af kind: Event lastTimestamp: "2026-07-02T21:31:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:31:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:15Z" name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw.18be9667f8e1a7e1 namespace: kserve-ci-e2e-test resourceVersion: "15605" uid: ee9c26a0-39e5-4da9-bcf4-2d54954c107b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: f252efa2-359f-4eb0-8a51-c2cf43f952af kind: Event lastTimestamp: "2026-07-02T21:31:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:31:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:15Z" name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw.18be9667f94fc385 namespace: kserve-ci-e2e-test resourceVersion: "15606" uid: f9801b87-f294-4283-89c7-d08a213c6dd8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: f252efa2-359f-4eb0-8a51-c2cf43f952af kind: Event lastTimestamp: "2026-07-02T21:31:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:31:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:15Z" name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw.18be9667f95a4cc1 namespace: kserve-ci-e2e-test resourceVersion: "15608" uid: e0c740a6-e83b-4ac0-bbcb-71894ef3cd53 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: f252efa2-359f-4eb0-8a51-c2cf43f952af kind: Event lastTimestamp: "2026-07-02T21:31:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:31:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:15Z" name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw.18be9667fcc12f03 namespace: kserve-ci-e2e-test resourceVersion: "15610" uid: 852b497b-a389-4a9b-be23-cfc82f867bdd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:31:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: f252efa2-359f-4eb0-8a51-c2cf43f952af kind: Event lastTimestamp: "2026-07-02T21:31:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:31:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:31:15Z" name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw.18be9667fd2d89e2 namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: f8870a93-6ae7-4886-93ab-f46b3728d422 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T21:31:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: f252efa2-359f-4eb0-8a51-c2cf43f952af kind: Event lastTimestamp: "2026-07-02T21:32:43Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:31:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:43Z" name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw.18be966845cee3ba namespace: kserve-ci-e2e-test resourceVersion: "16435" uid: 903036c5-5ddc-40c1-9a46-d64dac87d97b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: f252efa2-359f-4eb0-8a51-c2cf43f952af kind: Event lastTimestamp: "2026-07-02T21:32:42Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:42Z" name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw.18be967c205d54ee namespace: kserve-ci-e2e-test resourceVersion: "16408" uid: b79f2ae1-13e4-45bb-8473-ebeb64f27bb7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: f252efa2-359f-4eb0-8a51-c2cf43f952af kind: Event lastTimestamp: "2026-07-02T21:32:42Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:42Z" name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw.18be967c205dbb75 namespace: kserve-ci-e2e-test resourceVersion: "16410" uid: 181ccbac-0b03-4e78-bc20-32fd55d1fbe3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: f252efa2-359f-4eb0-8a51-c2cf43f952af kind: Event lastTimestamp: "2026-07-02T21:32:43Z" message: 'Readiness probe failed: Get "https://10.132.0.25:8643/healthz": dial tcp 10.132.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:32:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:43Z" name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw.18be967c4bfe9173 namespace: kserve-ci-e2e-test resourceVersion: "16434" uid: 2c9bf732-7582-4855-a6a2-3f05babfdc13 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857 namespace: kserve-ci-e2e-test resourceVersion: "15356" uid: 026b1879-7ee2-446d-ad21-ea422217cced kind: Event lastTimestamp: "2026-07-02T21:30:51Z" message: 'Created pod: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857-2lbrw' metadata: creationTimestamp: "2026-07-02T21:30:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:30:51Z" name: isvc-xgboost-graph-raw-c1b1e-predictor-6976897857.18be96626b9acb86 namespace: kserve-ci-e2e-test resourceVersion: "15373" uid: cb0bc903-7bb8-458f-9df6-9143253acdaa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-c1b1e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15350" uid: f6506051-84a1-4a16-811b-3d20c8331537 kind: Event lastTimestamp: "2026-07-02T21:30:51Z" message: Scaled up replica set isvc-xgboost-graph-raw-c1b1e-predictor-6976897857 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:30:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:30:51Z" name: isvc-xgboost-graph-raw-c1b1e-predictor.18be966267fbe577 namespace: kserve-ci-e2e-test resourceVersion: "15358" uid: 5b5a8cf6-e5bb-45b7-80b9-dd922842423f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:31:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-c1b1e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: 7018cab7-f4da-428e-a6ad-8ce47ece3bf0 kind: Event lastTimestamp: "2026-07-02T21:31:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:31:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:36Z" name: isvc-xgboost-graph-raw-c1b1e-predictor.18be9665e6969800 namespace: kserve-ci-e2e-test resourceVersion: "15778" uid: 26ebfd48-a89e-432d-99f0-1bd02ea22052 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:31:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-c1b1e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: 7018cab7-f4da-428e-a6ad-8ce47ece3bf0 kind: Event lastTimestamp: "2026-07-02T21:31:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:31:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:31:36Z" name: isvc-xgboost-graph-raw-c1b1e-predictor.18be9665e6fe944d namespace: kserve-ci-e2e-test resourceVersion: "15779" uid: 5ee1fa43-4b0b-4a6e-970d-54e3c5f8dae3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:31:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-c1b1e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15545" uid: 7018cab7-f4da-428e-a6ad-8ce47ece3bf0 kind: Event lastTimestamp: "2026-07-02T21:32:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:31:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:36Z" name: isvc-xgboost-graph-raw-c1b1e-predictor.18be9670632eba4b namespace: kserve-ci-e2e-test resourceVersion: "16261" uid: 9bcbd7cd-3bae-4b81-b0c8-9ce3d9ccc8c0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:31:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-c1b1e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15545" uid: 7018cab7-f4da-428e-a6ad-8ce47ece3bf0 kind: Event lastTimestamp: "2026-07-02T21:32:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:31:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:36Z" name: isvc-xgboost-graph-raw-c1b1e-predictor.18be96706396a3b1 namespace: kserve-ci-e2e-test resourceVersion: "16262" uid: 83324465-c3e1-4cee-88e0-419da66b20d8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T21:30:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-c1b1e namespace: kserve-ci-e2e-test resourceVersion: "15320" uid: 363a2d62-cf08-4a5e-a253-a06ceb494cf8 kind: Event lastTimestamp: "2026-07-02T21:32:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:30:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:32:23Z" name: isvc-xgboost-graph-raw-c1b1e.18be9662683f147e namespace: kserve-ci-e2e-test resourceVersion: "16130" uid: 1b0ed9ee-d6fc-4169-b393-99a4dcf5f79e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:30:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-c1b1e namespace: kserve-ci-e2e-test resourceVersion: "15320" uid: 363a2d62-cf08-4a5e-a253-a06ceb494cf8 kind: Event lastTimestamp: "2026-07-02T21:30:51Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-c1b1e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-c1b1e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:30:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:30:51Z" name: isvc-xgboost-graph-raw-c1b1e.18be966268d387d4 namespace: kserve-ci-e2e-test resourceVersion: "15363" uid: b1df9783-97db-4323-9cfd-99e82c0075e9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-c1b1e namespace: kserve-ci-e2e-test resourceVersion: "16128" uid: 363a2d62-cf08-4a5e-a253-a06ceb494cf8 kind: Event lastTimestamp: "2026-07-02T21:32:23Z" message: InferenceService [isvc-xgboost-graph-raw-c1b1e] is Ready metadata: creationTimestamp: "2026-07-02T21:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:32:23Z" name: isvc-xgboost-graph-raw-c1b1e.18be9677a9dee4a0 namespace: kserve-ci-e2e-test resourceVersion: "16129" uid: a8740e2f-e722-4dd6-ade9-ffe9e6d9b66d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:32:42.301229Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: 89e82f6f-f14e-4426-9c7d-9bc4fee4efb5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c to ip-10-0-135-227.ec2.internal metadata: creationTimestamp: "2026-07-02T21:32:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:32:42Z" name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c.18be967c1d8dd462 namespace: kserve-ci-e2e-test resourceVersion: "16387" uid: 711bd8cb-0a22-46fb-8159-c6fb4845850f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-558f78ddf7-rgj7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:42Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c namespace: kserve-ci-e2e-test resourceVersion: "16385" uid: 89e82f6f-f14e-4426-9c7d-9bc4fee4efb5 kind: Event lastTimestamp: "2026-07-02T21:32:42Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-hpa-af6d2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T21:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:42Z" name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c.18be967c280f6c73 namespace: kserve-ci-e2e-test resourceVersion: "16421" uid: fa3eaa94-78ca-4a99-bf7c-ea6f0faa5fee reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:43Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c namespace: kserve-ci-e2e-test resourceVersion: "16405" uid: 89e82f6f-f14e-4426-9c7d-9bc4fee4efb5 kind: Event lastTimestamp: "2026-07-02T21:32:43Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:32:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:32:43Z" name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c.18be967c5b564114 namespace: kserve-ci-e2e-test resourceVersion: "16441" uid: 5056f016-6e5d-4b4f-af26-8c822e81db52 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c namespace: kserve-ci-e2e-test resourceVersion: "16385" uid: 89e82f6f-f14e-4426-9c7d-9bc4fee4efb5 kind: Event lastTimestamp: "2026-07-02T21:32:43Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:33f82c067d520fe749f73749cf76b5935fe69bd9eb141504c6660698c7947743" already present on machine metadata: creationTimestamp: "2026-07-02T21:32:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:43Z" name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c.18be967c5c9a98dd namespace: kserve-ci-e2e-test resourceVersion: "16445" uid: 97f4a1c4-3277-40dd-8d48-f15115aa13f2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c namespace: kserve-ci-e2e-test resourceVersion: "16385" uid: 89e82f6f-f14e-4426-9c7d-9bc4fee4efb5 kind: Event lastTimestamp: "2026-07-02T21:32:43Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T21:32:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:43Z" name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c.18be967c5f36008b namespace: kserve-ci-e2e-test resourceVersion: "16446" uid: e4a68ac1-d627-4a46-8b08-009840a2c869 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c namespace: kserve-ci-e2e-test resourceVersion: "16385" uid: 89e82f6f-f14e-4426-9c7d-9bc4fee4efb5 kind: Event lastTimestamp: "2026-07-02T21:32:43Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T21:32:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:43Z" name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c.18be967c5f9766b4 namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: d9438598-76d5-4264-825b-066cb21e4358 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c namespace: kserve-ci-e2e-test resourceVersion: "16385" uid: 89e82f6f-f14e-4426-9c7d-9bc4fee4efb5 kind: Event lastTimestamp: "2026-07-02T21:32:47Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-07-02T21:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:47Z" name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c.18be967d51e4c14b namespace: kserve-ci-e2e-test resourceVersion: "16493" uid: 6ab653af-01f3-40d9-a090-e7cabe43382b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c namespace: kserve-ci-e2e-test resourceVersion: "16385" uid: 89e82f6f-f14e-4426-9c7d-9bc4fee4efb5 kind: Event lastTimestamp: "2026-07-02T21:32:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:47Z" name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c.18be967d5577a081 namespace: kserve-ci-e2e-test resourceVersion: "16502" uid: 3ee2dbae-1eb9-47a3-8420-bb583de518fe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c namespace: kserve-ci-e2e-test resourceVersion: "16385" uid: 89e82f6f-f14e-4426-9c7d-9bc4fee4efb5 kind: Event lastTimestamp: "2026-07-02T21:32:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:47Z" name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c.18be967d55f9c9ce namespace: kserve-ci-e2e-test resourceVersion: "16503" uid: 39814549-e14c-49b6-b498-2afe0ff45c41 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c namespace: kserve-ci-e2e-test resourceVersion: "16385" uid: 89e82f6f-f14e-4426-9c7d-9bc4fee4efb5 kind: Event lastTimestamp: "2026-07-02T21:32:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:47Z" name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c.18be967d5604b93b namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: abb9b3af-c6e5-4328-8c55-6fae4e9c27ed reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c namespace: kserve-ci-e2e-test resourceVersion: "16385" uid: 89e82f6f-f14e-4426-9c7d-9bc4fee4efb5 kind: Event lastTimestamp: "2026-07-02T21:32:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:47Z" name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c.18be967d58acc290 namespace: kserve-ci-e2e-test resourceVersion: "16509" uid: 1eb0f56a-f63e-41b0-85fb-5790a18cdadd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c namespace: kserve-ci-e2e-test resourceVersion: "16385" uid: 89e82f6f-f14e-4426-9c7d-9bc4fee4efb5 kind: Event lastTimestamp: "2026-07-02T21:32:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:47Z" name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c.18be967d5931f237 namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: e1714e84-643c-4296-a5b8-4fa551083ee4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-02T21:32:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c namespace: kserve-ci-e2e-test resourceVersion: "16385" uid: 89e82f6f-f14e-4426-9c7d-9bc4fee4efb5 kind: Event lastTimestamp: "2026-07-02T21:34:35Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:32:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:35Z" name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c.18be967dc9f4f9b9 namespace: kserve-ci-e2e-test resourceVersion: "17464" uid: 362583a3-c96f-43e9-8141-1ecf659a253d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c namespace: kserve-ci-e2e-test resourceVersion: "16385" uid: 89e82f6f-f14e-4426-9c7d-9bc4fee4efb5 kind: Event lastTimestamp: "2026-07-02T21:34:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:34:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:32Z" name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c.18be9695cc6dfbcb namespace: kserve-ci-e2e-test resourceVersion: "17423" uid: 0ae26db1-5e25-4b7e-84d6-e75a27ab8b62 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c namespace: kserve-ci-e2e-test resourceVersion: "16385" uid: 89e82f6f-f14e-4426-9c7d-9bc4fee4efb5 kind: Event lastTimestamp: "2026-07-02T21:34:32Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:34:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:32Z" name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c.18be9695cc6e6dc7 namespace: kserve-ci-e2e-test resourceVersion: "17425" uid: 67876f3b-f609-478b-933e-c81549385abc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c namespace: kserve-ci-e2e-test resourceVersion: "16385" uid: 89e82f6f-f14e-4426-9c7d-9bc4fee4efb5 kind: Event lastTimestamp: "2026-07-02T21:34:35Z" message: 'Readiness probe failed: Get "https://10.132.0.27:8643/healthz": dial tcp 10.132.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:34:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:35Z" name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c.18be9696783bbb7f namespace: kserve-ci-e2e-test resourceVersion: "17463" uid: 052c9108-54d6-4468-bcf7-5ed3c801cce2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57 namespace: kserve-ci-e2e-test resourceVersion: "16372" uid: fb43dca9-f0ef-4e00-bdb6-f53229c8ee61 kind: Event lastTimestamp: "2026-07-02T21:32:42Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57-59x9c' metadata: creationTimestamp: "2026-07-02T21:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:42Z" name: isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57.18be967c1d23d9bf namespace: kserve-ci-e2e-test resourceVersion: "16384" uid: 94cfd19b-6d25-4d48-8f43-0fb2b2bf65ce reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-af6d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16366" uid: 3073f0f1-4900-4d6b-a2ef-169cb85c329d kind: Event lastTimestamp: "2026-07-02T21:32:42Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-af6d2-predictor-69bcd97d57 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:42Z" name: isvc-xgboost-graph-raw-hpa-af6d2-predictor.18be967c1a41ec20 namespace: kserve-ci-e2e-test resourceVersion: "16373" uid: 73369b5d-aa0b-407d-a90b-492df63465b6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:32:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-af6d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16371" uid: dcc896bc-106c-4b82-9fac-214f084e89cd kind: Event lastTimestamp: "2026-07-02T21:33:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:32:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:33:12Z" name: isvc-xgboost-graph-raw-hpa-af6d2-predictor.18be967f98fee4e0 namespace: kserve-ci-e2e-test resourceVersion: "16709" uid: 15ce7eb8-47fc-4ed3-9c3d-5329d323c628 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:32:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-af6d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16371" uid: dcc896bc-106c-4b82-9fac-214f084e89cd kind: Event lastTimestamp: "2026-07-02T21:33:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:32:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:33:12Z" name: isvc-xgboost-graph-raw-hpa-af6d2-predictor.18be967f996504c8 namespace: kserve-ci-e2e-test resourceVersion: "16710" uid: 0f462873-5232-42ce-aee4-0e31c2903f53 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:33:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-af6d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16599" uid: dcc896bc-106c-4b82-9fac-214f084e89cd kind: Event lastTimestamp: "2026-07-02T21:34:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:33:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:34:12Z" name: isvc-xgboost-graph-raw-hpa-af6d2-predictor.18be968696ae68f5 namespace: kserve-ci-e2e-test resourceVersion: "17157" uid: 78744580-d3a7-48d0-a03e-66482d5a9042 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:33:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-af6d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16599" uid: dcc896bc-106c-4b82-9fac-214f084e89cd kind: Event lastTimestamp: "2026-07-02T21:34:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:33:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:34:12Z" name: isvc-xgboost-graph-raw-hpa-af6d2-predictor.18be96869710c23d namespace: kserve-ci-e2e-test resourceVersion: "17158" uid: 7e06b5f6-39d9-40d4-9200-3465ad5efa67 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T21:32:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-af6d2 namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: 5d62ca68-915e-4655-b33b-f39fd64a1a2a kind: Event lastTimestamp: "2026-07-02T21:33:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:33:45Z" name: isvc-xgboost-graph-raw-hpa-af6d2.18be967c1aab090a namespace: kserve-ci-e2e-test resourceVersion: "16963" uid: 2624f4fc-8064-480d-bfad-07e87b51dd74 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-af6d2 namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: 5d62ca68-915e-4655-b33b-f39fd64a1a2a kind: Event lastTimestamp: "2026-07-02T21:32:42Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-hpa-af6d2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-hpa-af6d2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:32:42Z" name: isvc-xgboost-graph-raw-hpa-af6d2.18be967c1b0ad1a4 namespace: kserve-ci-e2e-test resourceVersion: "16378" uid: 3e250096-3e15-4321-9293-2e375e1a60b0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:33:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-af6d2 namespace: kserve-ci-e2e-test resourceVersion: "16961" uid: 5d62ca68-915e-4655-b33b-f39fd64a1a2a kind: Event lastTimestamp: "2026-07-02T21:33:45Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-af6d2] is Ready metadata: creationTimestamp: "2026-07-02T21:33:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:33:45Z" name: isvc-xgboost-graph-raw-hpa-af6d2.18be968ad98a4afa namespace: kserve-ci-e2e-test resourceVersion: "16962" uid: bd14251e-bb91-4317-9d1d-df63d9e0cfc8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:34:32.471378Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-1f359-predictor-864d8c9886-4cb52 namespace: kserve-ci-e2e-test resourceVersion: "17377" uid: ceb934c5-0868-411c-bb37-3b3f3f2bf665 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-1f359-predictor-864d8c9886-4cb52 to ip-10-0-135-227.ec2.internal metadata: creationTimestamp: "2026-07-02T21:34:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:34:32Z" name: message-dumper-raw-1f359-predictor-864d8c9886-4cb52.18be9695c434e255 namespace: kserve-ci-e2e-test resourceVersion: "17381" uid: c57ee338-c918-4386-9acf-a134746e0c80 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-558f78ddf7-rgj7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:32Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-1f359-predictor-864d8c9886-4cb52 namespace: kserve-ci-e2e-test resourceVersion: "17379" uid: ceb934c5-0868-411c-bb37-3b3f3f2bf665 kind: Event lastTimestamp: "2026-07-02T21:34:32Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "message-dumper-raw-1f359-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T21:34:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:32Z" name: message-dumper-raw-1f359-predictor-864d8c9886-4cb52.18be9695d5139566 namespace: kserve-ci-e2e-test resourceVersion: "17428" uid: ae2cb228-bb62-4c2d-9a19-b4cfe561141f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:33Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-1f359-predictor-864d8c9886-4cb52 namespace: kserve-ci-e2e-test resourceVersion: "17387" uid: ceb934c5-0868-411c-bb37-3b3f3f2bf665 kind: Event lastTimestamp: "2026-07-02T21:34:33Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:34:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:34:33Z" name: message-dumper-raw-1f359-predictor-864d8c9886-4cb52.18be96960169f998 namespace: kserve-ci-e2e-test resourceVersion: "17437" uid: 33313450-c93b-4c50-a559-9049e23453b2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-1f359-predictor-864d8c9886-4cb52 namespace: kserve-ci-e2e-test resourceVersion: "17379" uid: ceb934c5-0868-411c-bb37-3b3f3f2bf665 kind: Event lastTimestamp: "2026-07-02T21:34:33Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-02T21:34:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:33Z" name: message-dumper-raw-1f359-predictor-864d8c9886-4cb52.18be969602d7b7ea namespace: kserve-ci-e2e-test resourceVersion: "17439" uid: ecc807df-81a6-42c4-a9fa-8b7b0e7df0a7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-1f359-predictor-864d8c9886-4cb52 namespace: kserve-ci-e2e-test resourceVersion: "17379" uid: ceb934c5-0868-411c-bb37-3b3f3f2bf665 kind: Event lastTimestamp: "2026-07-02T21:34:34Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.338s (1.338s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-02T21:34:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:34Z" name: message-dumper-raw-1f359-predictor-864d8c9886-4cb52.18be9696529afa88 namespace: kserve-ci-e2e-test resourceVersion: "17452" uid: 8b00d423-18d1-4f8b-ae93-1158ebbb2d99 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-1f359-predictor-864d8c9886-4cb52 namespace: kserve-ci-e2e-test resourceVersion: "17379" uid: ceb934c5-0868-411c-bb37-3b3f3f2bf665 kind: Event lastTimestamp: "2026-07-02T21:34:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:34:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:34Z" name: message-dumper-raw-1f359-predictor-864d8c9886-4cb52.18be96965611b2d1 namespace: kserve-ci-e2e-test resourceVersion: "17454" uid: d0d78d73-8915-417c-98fb-63992120db7f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-1f359-predictor-864d8c9886-4cb52 namespace: kserve-ci-e2e-test resourceVersion: "17379" uid: ceb934c5-0868-411c-bb37-3b3f3f2bf665 kind: Event lastTimestamp: "2026-07-02T21:34:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:34:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:34Z" name: message-dumper-raw-1f359-predictor-864d8c9886-4cb52.18be96965684677e namespace: kserve-ci-e2e-test resourceVersion: "17455" uid: c9c2c701-47b2-4359-b119-72c16d0ebe3c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-1f359-predictor-864d8c9886-4cb52 namespace: kserve-ci-e2e-test resourceVersion: "17379" uid: ceb934c5-0868-411c-bb37-3b3f3f2bf665 kind: Event lastTimestamp: "2026-07-02T21:34:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:34:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:34Z" name: message-dumper-raw-1f359-predictor-864d8c9886-4cb52.18be9696568e3c5e namespace: kserve-ci-e2e-test resourceVersion: "17456" uid: 916a916e-9532-4232-83b4-e44760312653 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-1f359-predictor-864d8c9886-4cb52 namespace: kserve-ci-e2e-test resourceVersion: "17379" uid: ceb934c5-0868-411c-bb37-3b3f3f2bf665 kind: Event lastTimestamp: "2026-07-02T21:34:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:34:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:34Z" name: message-dumper-raw-1f359-predictor-864d8c9886-4cb52.18be969659e0f6fb namespace: kserve-ci-e2e-test resourceVersion: "17457" uid: c573ae98-80ad-4ced-8cba-788f4022c122 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-1f359-predictor-864d8c9886-4cb52 namespace: kserve-ci-e2e-test resourceVersion: "17379" uid: ceb934c5-0868-411c-bb37-3b3f3f2bf665 kind: Event lastTimestamp: "2026-07-02T21:34:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:34:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:34Z" name: message-dumper-raw-1f359-predictor-864d8c9886-4cb52.18be96965a500fee namespace: kserve-ci-e2e-test resourceVersion: "17458" uid: 6b52f268-6af2-4eaf-8bb8-41793f6cde57 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:36:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-1f359-predictor-864d8c9886-4cb52 namespace: kserve-ci-e2e-test resourceVersion: "17379" uid: ceb934c5-0868-411c-bb37-3b3f3f2bf665 kind: Event lastTimestamp: "2026-07-02T21:36:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:36:27Z" name: message-dumper-raw-1f359-predictor-864d8c9886-4cb52.18be96b0a2afdae9 namespace: kserve-ci-e2e-test resourceVersion: "18446" uid: f738f105-4214-42e1-bc75-fcabe8f0c3a1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:36:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-1f359-predictor-864d8c9886-4cb52 namespace: kserve-ci-e2e-test resourceVersion: "17379" uid: ceb934c5-0868-411c-bb37-3b3f3f2bf665 kind: Event lastTimestamp: "2026-07-02T21:36:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:36:27Z" name: message-dumper-raw-1f359-predictor-864d8c9886-4cb52.18be96b0a2b0e983 namespace: kserve-ci-e2e-test resourceVersion: "18448" uid: cfa12a83-2979-445a-9bf7-0c3d454b0247 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-1f359-predictor-864d8c9886 namespace: kserve-ci-e2e-test resourceVersion: "17366" uid: f892d34d-ded3-475b-a0b8-ab4504325876 kind: Event lastTimestamp: "2026-07-02T21:34:32Z" message: 'Created pod: message-dumper-raw-1f359-predictor-864d8c9886-4cb52' metadata: creationTimestamp: "2026-07-02T21:34:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:34:32Z" name: message-dumper-raw-1f359-predictor-864d8c9886.18be9695c3c622fc namespace: kserve-ci-e2e-test resourceVersion: "17378" uid: 0b7e5176-3e91-4f0c-9680-54d18c2da5f6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-1f359-predictor namespace: kserve-ci-e2e-test resourceVersion: "17364" uid: f65fedde-a99e-4f91-b7a1-2c050a64f3f4 kind: Event lastTimestamp: "2026-07-02T21:34:32Z" message: Scaled up replica set message-dumper-raw-1f359-predictor-864d8c9886 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:34:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:34:32Z" name: message-dumper-raw-1f359-predictor.18be9695c0c09066 namespace: kserve-ci-e2e-test resourceVersion: "17367" uid: e510180a-1e59-4798-b83c-03b6285644a6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:34:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-1f359-predictor namespace: kserve-ci-e2e-test resourceVersion: "17365" uid: dc6af5d6-e9d2-4361-8b27-c36477375957 kind: Event lastTimestamp: "2026-07-02T21:35:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:34:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:35:17Z" name: message-dumper-raw-1f359-predictor.18be96993fd154cd namespace: kserve-ci-e2e-test resourceVersion: "17864" uid: 3a9f8128-8a0b-4bb2-b07f-05ee51832479 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:34:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-1f359-predictor namespace: kserve-ci-e2e-test resourceVersion: "17365" uid: dc6af5d6-e9d2-4361-8b27-c36477375957 kind: Event lastTimestamp: "2026-07-02T21:35:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:34:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:35:17Z" name: message-dumper-raw-1f359-predictor.18be9699403aaffa namespace: kserve-ci-e2e-test resourceVersion: "17865" uid: 523dd3ea-685d-431c-9bee-36645de8e1ca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T21:34:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-1f359 namespace: kserve-ci-e2e-test resourceVersion: "17357" uid: 65efae78-febd-41b3-81f3-6f00f9b7d794 kind: Event lastTimestamp: "2026-07-02T21:34:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:34:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:34:44Z" name: message-dumper-raw-1f359.18be9695c0bdee09 namespace: kserve-ci-e2e-test resourceVersion: "17544" uid: 5b57d7ba-30bc-44b9-b0a9-c279c5e39e1a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-1f359 namespace: kserve-ci-e2e-test resourceVersion: "17357" uid: 65efae78-febd-41b3-81f3-6f00f9b7d794 kind: Event lastTimestamp: "2026-07-02T21:34:32Z" message: 'Failed to update status for InferenceService "message-dumper-raw-1f359": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-1f359": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:34:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:34:32Z" name: message-dumper-raw-1f359.18be9695c1173f77 namespace: kserve-ci-e2e-test resourceVersion: "17371" uid: fdefe50a-bf4a-4ce8-914f-667d05083c4e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-1f359 namespace: kserve-ci-e2e-test resourceVersion: "17542" uid: 65efae78-febd-41b3-81f3-6f00f9b7d794 kind: Event lastTimestamp: "2026-07-02T21:34:44Z" message: InferenceService [message-dumper-raw-1f359] is Ready metadata: creationTimestamp: "2026-07-02T21:34:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:34:44Z" name: message-dumper-raw-1f359.18be969875e887d2 namespace: kserve-ci-e2e-test resourceVersion: "17543" uid: 22c86400-52dd-421f-9f96-cf72e8538206 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:32:31.880863Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-c1b1e-5f996d4464-flxrp namespace: kserve-ci-e2e-test resourceVersion: "16198" uid: f660bd15-3bad-424e-bc3f-69fa51f6090b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-c1b1e-5f996d4464-flxrp to ip-10-0-133-168.ec2.internal metadata: creationTimestamp: "2026-07-02T21:32:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:32:31Z" name: model-chainer-raw-c1b1e-5f996d4464-flxrp.18be9679b073d7e5 namespace: kserve-ci-e2e-test resourceVersion: "16203" uid: c9fb1324-5e37-4574-9213-cde8a43383e9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-558f78ddf7-rgj7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:32Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-c1b1e-5f996d4464-flxrp namespace: kserve-ci-e2e-test resourceVersion: "16201" uid: f660bd15-3bad-424e-bc3f-69fa51f6090b kind: Event lastTimestamp: "2026-07-02T21:32:32Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-c1b1e-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T21:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:32Z" name: model-chainer-raw-c1b1e-5f996d4464-flxrp.18be9679bdaeb923 namespace: kserve-ci-e2e-test resourceVersion: "16215" uid: bfcaf672-25b6-4c51-ba19-b31c26d02444 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-168.ec2.internal source: component: kubelet host: ip-10-0-133-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:32Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-c1b1e-5f996d4464-flxrp namespace: kserve-ci-e2e-test resourceVersion: "16207" uid: f660bd15-3bad-424e-bc3f-69fa51f6090b kind: Event lastTimestamp: "2026-07-02T21:32:32Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:32:32Z" name: model-chainer-raw-c1b1e-5f996d4464-flxrp.18be9679ee32297d namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: 748bca94-eebb-4af8-8664-eb20c1f27959 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c1b1e} kind: Pod name: model-chainer-raw-c1b1e-5f996d4464-flxrp namespace: kserve-ci-e2e-test resourceVersion: "16201" uid: f660bd15-3bad-424e-bc3f-69fa51f6090b kind: Event lastTimestamp: "2026-07-02T21:32:32Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:a0ae04c8af985f4fab778b415e570561f00fd874023f1d5c87c716054c3fb7d5" metadata: creationTimestamp: "2026-07-02T21:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:32Z" name: model-chainer-raw-c1b1e-5f996d4464-flxrp.18be9679efc34671 namespace: kserve-ci-e2e-test resourceVersion: "16226" uid: 75e89eb1-df2f-4688-887a-3d9ce0735bd5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-168.ec2.internal source: component: kubelet host: ip-10-0-133-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c1b1e} kind: Pod name: model-chainer-raw-c1b1e-5f996d4464-flxrp namespace: kserve-ci-e2e-test resourceVersion: "16201" uid: f660bd15-3bad-424e-bc3f-69fa51f6090b kind: Event lastTimestamp: "2026-07-02T21:32:35Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:a0ae04c8af985f4fab778b415e570561f00fd874023f1d5c87c716054c3fb7d5" in 2.301s (2.301s including waiting). Image size: 218072672 bytes.' metadata: creationTimestamp: "2026-07-02T21:32:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:35Z" name: model-chainer-raw-c1b1e-5f996d4464-flxrp.18be967a78f45ab0 namespace: kserve-ci-e2e-test resourceVersion: "16243" uid: 76538109-63e2-46e6-b4ce-5e9282535633 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-168.ec2.internal source: component: kubelet host: ip-10-0-133-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c1b1e} kind: Pod name: model-chainer-raw-c1b1e-5f996d4464-flxrp namespace: kserve-ci-e2e-test resourceVersion: "16201" uid: f660bd15-3bad-424e-bc3f-69fa51f6090b kind: Event lastTimestamp: "2026-07-02T21:32:35Z" message: 'Created container: model-chainer-raw-c1b1e' metadata: creationTimestamp: "2026-07-02T21:32:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:35Z" name: model-chainer-raw-c1b1e-5f996d4464-flxrp.18be967a7c3f3f9b namespace: kserve-ci-e2e-test resourceVersion: "16244" uid: 2505219e-eb06-43b2-9cb8-e1ff11a0ef1b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-168.ec2.internal source: component: kubelet host: ip-10-0-133-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c1b1e} kind: Pod name: model-chainer-raw-c1b1e-5f996d4464-flxrp namespace: kserve-ci-e2e-test resourceVersion: "16201" uid: f660bd15-3bad-424e-bc3f-69fa51f6090b kind: Event lastTimestamp: "2026-07-02T21:32:35Z" message: Started container model-chainer-raw-c1b1e metadata: creationTimestamp: "2026-07-02T21:32:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:35Z" name: model-chainer-raw-c1b1e-5f996d4464-flxrp.18be967a7cba2332 namespace: kserve-ci-e2e-test resourceVersion: "16245" uid: 43225e70-edac-4134-bce0-d19c1e8ce951 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-168.ec2.internal source: component: kubelet host: ip-10-0-133-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c1b1e} kind: Pod name: model-chainer-raw-c1b1e-5f996d4464-flxrp namespace: kserve-ci-e2e-test resourceVersion: "16201" uid: f660bd15-3bad-424e-bc3f-69fa51f6090b kind: Event lastTimestamp: "2026-07-02T21:32:41Z" message: Stopping container model-chainer-raw-c1b1e metadata: creationTimestamp: "2026-07-02T21:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:32:41Z" name: model-chainer-raw-c1b1e-5f996d4464-flxrp.18be967c08d01d22 namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: 6a02f3d5-5558-4db4-9932-266d517603f7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-168.ec2.internal source: component: kubelet host: ip-10-0-133-168.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:32:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c1b1e} kind: Pod name: model-chainer-raw-c1b1e-5f996d4464-flxrp namespace: kserve-ci-e2e-test resourceVersion: "16201" uid: f660bd15-3bad-424e-bc3f-69fa51f6090b kind: Event lastTimestamp: "2026-07-02T21:33:06Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T21:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:33:06Z" name: model-chainer-raw-c1b1e-5f996d4464-flxrp.18be967c092b459b namespace: kserve-ci-e2e-test resourceVersion: "16668" uid: 41039af1-542c-4b1a-935b-baf374c5880b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-168.ec2.internal source: component: kubelet host: ip-10-0-133-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:33:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c1b1e} kind: Pod name: model-chainer-raw-c1b1e-5f996d4464-flxrp namespace: kserve-ci-e2e-test resourceVersion: "16201" uid: f660bd15-3bad-424e-bc3f-69fa51f6090b kind: Event lastTimestamp: "2026-07-02T21:33:11Z" message: 'Readiness probe failed: Get "https://10.134.0.20:8080/readyz": dial tcp 10.134.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:33:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:33:11Z" name: model-chainer-raw-c1b1e-5f996d4464-flxrp.18be9683053a7ad8 namespace: kserve-ci-e2e-test resourceVersion: "16702" uid: dcc072c8-2f23-4803-ac5e-a3d61e199031 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-168.ec2.internal source: component: kubelet host: ip-10-0-133-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-c1b1e-5f996d4464 namespace: kserve-ci-e2e-test resourceVersion: "16195" uid: 4b44e00b-0dba-46d2-92ce-90d76e32643c kind: Event lastTimestamp: "2026-07-02T21:32:31Z" message: 'Created pod: model-chainer-raw-c1b1e-5f996d4464-flxrp' metadata: creationTimestamp: "2026-07-02T21:32:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:31Z" name: model-chainer-raw-c1b1e-5f996d4464.18be9679aff90cd9 namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: f243facd-2d3a-44de-b174-fbd17e3ce9e1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-c1b1e namespace: kserve-ci-e2e-test resourceVersion: "16193" uid: 488d924a-5b54-4771-ade7-fd174279cf97 kind: Event lastTimestamp: "2026-07-02T21:32:31Z" message: Scaled up replica set model-chainer-raw-c1b1e-5f996d4464 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:32:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:32:31Z" name: model-chainer-raw-c1b1e.18be9679aed86567 namespace: kserve-ci-e2e-test resourceVersion: "16196" uid: f7825c11-1a08-43bb-b9f0-defd18dfb944 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:32:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-c1b1e namespace: kserve-ci-e2e-test resourceVersion: "16209" uid: 230e4794-2653-48d3-ae85-0c69890cc5b3 kind: Event lastTimestamp: "2026-07-02T21:32:31Z" message: InferenceGraph [model-chainer-raw-c1b1e] is Ready metadata: creationTimestamp: "2026-07-02T21:32:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:32:31Z" name: model-chainer-raw-c1b1e.18be9679b310f656 namespace: kserve-ci-e2e-test resourceVersion: "16211" uid: 6d3e24e6-a4ae-4dfc-9672-a527382022d5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:34:22.202325Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-af6d2-6cc8cfdbb5-prxz4 namespace: kserve-ci-e2e-test resourceVersion: "17242" uid: a437d0de-4e8a-481f-a008-e8f0b374ecd1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-af6d2-6cc8cfdbb5-prxz4 to ip-10-0-133-168.ec2.internal metadata: creationTimestamp: "2026-07-02T21:34:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:34:22Z" name: model-chainer-raw-hpa-af6d2-6cc8cfdbb5-prxz4.18be9693601f8358 namespace: kserve-ci-e2e-test resourceVersion: "17247" uid: aea1d96f-5c42-44f9-b7ca-27a5a4c88de7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-558f78ddf7-rgj7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:22Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-af6d2-6cc8cfdbb5-prxz4 namespace: kserve-ci-e2e-test resourceVersion: "17251" uid: a437d0de-4e8a-481f-a008-e8f0b374ecd1 kind: Event lastTimestamp: "2026-07-02T21:34:22Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:34:22Z" name: model-chainer-raw-hpa-af6d2-6cc8cfdbb5-prxz4.18be969379c8d3e2 namespace: kserve-ci-e2e-test resourceVersion: "17262" uid: c02c0cca-4477-4a2d-8e57-3536b3be6e1d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-af6d2} kind: Pod name: model-chainer-raw-hpa-af6d2-6cc8cfdbb5-prxz4 namespace: kserve-ci-e2e-test resourceVersion: "17245" uid: a437d0de-4e8a-481f-a008-e8f0b374ecd1 kind: Event lastTimestamp: "2026-07-02T21:34:22Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a0ae04c8af985f4fab778b415e570561f00fd874023f1d5c87c716054c3fb7d5" already present on machine metadata: creationTimestamp: "2026-07-02T21:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:22Z" name: model-chainer-raw-hpa-af6d2-6cc8cfdbb5-prxz4.18be96937b68018b namespace: kserve-ci-e2e-test resourceVersion: "17264" uid: 8291f2af-f49e-4658-a2d6-f85afd9a4ece reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-168.ec2.internal source: component: kubelet host: ip-10-0-133-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-af6d2} kind: Pod name: model-chainer-raw-hpa-af6d2-6cc8cfdbb5-prxz4 namespace: kserve-ci-e2e-test resourceVersion: "17245" uid: a437d0de-4e8a-481f-a008-e8f0b374ecd1 kind: Event lastTimestamp: "2026-07-02T21:34:22Z" message: 'Created container: model-chainer-raw-hpa-af6d2' metadata: creationTimestamp: "2026-07-02T21:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:22Z" name: model-chainer-raw-hpa-af6d2-6cc8cfdbb5-prxz4.18be96937e3aba42 namespace: kserve-ci-e2e-test resourceVersion: "17265" uid: 3c7ed2d0-16ff-48ed-aefa-adb34d50cf53 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-168.ec2.internal source: component: kubelet host: ip-10-0-133-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-af6d2} kind: Pod name: model-chainer-raw-hpa-af6d2-6cc8cfdbb5-prxz4 namespace: kserve-ci-e2e-test resourceVersion: "17245" uid: a437d0de-4e8a-481f-a008-e8f0b374ecd1 kind: Event lastTimestamp: "2026-07-02T21:34:22Z" message: Started container model-chainer-raw-hpa-af6d2 metadata: creationTimestamp: "2026-07-02T21:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:22Z" name: model-chainer-raw-hpa-af6d2-6cc8cfdbb5-prxz4.18be96937ed3c57b namespace: kserve-ci-e2e-test resourceVersion: "17266" uid: 4301d3b9-2259-485f-8f24-90a3ef76339b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-168.ec2.internal source: component: kubelet host: ip-10-0-133-168.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-af6d2} kind: Pod name: model-chainer-raw-hpa-af6d2-6cc8cfdbb5-prxz4 namespace: kserve-ci-e2e-test resourceVersion: "17245" uid: a437d0de-4e8a-481f-a008-e8f0b374ecd1 kind: Event lastTimestamp: "2026-07-02T21:34:32Z" message: Stopping container model-chainer-raw-hpa-af6d2 metadata: creationTimestamp: "2026-07-02T21:34:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:32Z" name: model-chainer-raw-hpa-af6d2-6cc8cfdbb5-prxz4.18be9695b7d93f27 namespace: kserve-ci-e2e-test resourceVersion: "17351" uid: 4ffa5cd3-07da-40a2-8dbe-ddbeb95bee2e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-168.ec2.internal source: component: kubelet host: ip-10-0-133-168.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T21:34:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-af6d2} kind: Pod name: model-chainer-raw-hpa-af6d2-6cc8cfdbb5-prxz4 namespace: kserve-ci-e2e-test resourceVersion: "17245" uid: a437d0de-4e8a-481f-a008-e8f0b374ecd1 kind: Event lastTimestamp: "2026-07-02T21:34:59Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T21:34:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:34:59Z" name: model-chainer-raw-hpa-af6d2-6cc8cfdbb5-prxz4.18be969638b0727d namespace: kserve-ci-e2e-test resourceVersion: "17722" uid: 76f60ce8-a918-4199-9f28-1bc9955fc10d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-168.ec2.internal source: component: kubelet host: ip-10-0-133-168.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-af6d2-6cc8cfdbb5 namespace: kserve-ci-e2e-test resourceVersion: "17239" uid: bf871347-6cca-466d-936b-6f3c52db88e2 kind: Event lastTimestamp: "2026-07-02T21:34:22Z" message: 'Created pod: model-chainer-raw-hpa-af6d2-6cc8cfdbb5-prxz4' metadata: creationTimestamp: "2026-07-02T21:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:34:22Z" name: model-chainer-raw-hpa-af6d2-6cc8cfdbb5.18be96935fb5b2ad namespace: kserve-ci-e2e-test resourceVersion: "17244" uid: 427a1f04-fe95-4163-bf9a-b03e2e48b558 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-af6d2 namespace: kserve-ci-e2e-test resourceVersion: "17236" uid: df014101-4383-4225-8fb4-a2c0e78c8846 kind: Event lastTimestamp: "2026-07-02T21:34:22Z" message: Scaled up replica set model-chainer-raw-hpa-af6d2-6cc8cfdbb5 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:34:22Z" name: model-chainer-raw-hpa-af6d2.18be96935e91a25d namespace: kserve-ci-e2e-test resourceVersion: "17240" uid: 25bf7590-beb4-41c5-a797-323a13024c57 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:34:22Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-af6d2 namespace: kserve-ci-e2e-test resourceVersion: "17253" uid: 1edf631f-919d-4430-8665-2ef2dfb6e030 kind: Event lastTimestamp: "2026-07-02T21:34:22Z" message: InferenceGraph [model-chainer-raw-hpa-af6d2] is Ready metadata: creationTimestamp: "2026-07-02T21:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:34:22Z" name: model-chainer-raw-hpa-af6d2.18be9693645ee2e2 namespace: kserve-ci-e2e-test resourceVersion: "17255" uid: 599ebe14-115e-490e-9f7c-d7f2586f1f35 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:41:04.737256Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt namespace: kserve-ci-e2e-test resourceVersion: "20749" uid: bed40e68-dadf-47b8-b904-47226e33c87a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt to ip-10-0-135-227.ec2.internal metadata: creationTimestamp: "2026-07-02T21:41:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:41:04Z" name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt.18be96f11913280a namespace: kserve-ci-e2e-test resourceVersion: "20755" uid: 15288432-54e7-4c70-8c3e-21c60e43ec0d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-558f78ddf7-rgj7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:05Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: bed40e68-dadf-47b8-b904-47226e33c87a kind: Event lastTimestamp: "2026-07-02T21:41:05Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:41:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:41:05Z" name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt.18be96f1325b58e5 namespace: kserve-ci-e2e-test resourceVersion: "20772" uid: ed611e1f-81bc-4070-b2a4-cb908623b02b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt namespace: kserve-ci-e2e-test resourceVersion: "20752" uid: bed40e68-dadf-47b8-b904-47226e33c87a kind: Event lastTimestamp: "2026-07-02T21:41:05Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:33f82c067d520fe749f73749cf76b5935fe69bd9eb141504c6660698c7947743" already present on machine metadata: creationTimestamp: "2026-07-02T21:41:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:05Z" name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt.18be96f134448504 namespace: kserve-ci-e2e-test resourceVersion: "20774" uid: 9bcbd5f2-aa86-46fb-9970-2550409b26e9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt namespace: kserve-ci-e2e-test resourceVersion: "20752" uid: bed40e68-dadf-47b8-b904-47226e33c87a kind: Event lastTimestamp: "2026-07-02T21:41:05Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T21:41:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:05Z" name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt.18be96f137320474 namespace: kserve-ci-e2e-test resourceVersion: "20775" uid: ccdb09ff-1e5b-48b9-bac4-7bc9874a4a96 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt namespace: kserve-ci-e2e-test resourceVersion: "20752" uid: bed40e68-dadf-47b8-b904-47226e33c87a kind: Event lastTimestamp: "2026-07-02T21:41:05Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T21:41:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:05Z" name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt.18be96f1379ae88f namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: d6933751-0b6f-4a30-8b42-ce8f5630224b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt namespace: kserve-ci-e2e-test resourceVersion: "20752" uid: bed40e68-dadf-47b8-b904-47226e33c87a kind: Event lastTimestamp: "2026-07-02T21:41:09Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:41:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:09Z" name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt.18be96f23965d843 namespace: kserve-ci-e2e-test resourceVersion: "20811" uid: d77386db-9c7b-4760-a739-b0411b59f500 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt namespace: kserve-ci-e2e-test resourceVersion: "20752" uid: bed40e68-dadf-47b8-b904-47226e33c87a kind: Event lastTimestamp: "2026-07-02T21:41:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:41:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:09Z" name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt.18be96f23c6d1072 namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: 52916fe1-1c46-4dad-b1e5-15892f0744b3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt namespace: kserve-ci-e2e-test resourceVersion: "20752" uid: bed40e68-dadf-47b8-b904-47226e33c87a kind: Event lastTimestamp: "2026-07-02T21:41:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:41:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:09Z" name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt.18be96f23ce61d1b namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: c08b69a5-e0aa-4ac0-a2fd-52b7df83a498 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt namespace: kserve-ci-e2e-test resourceVersion: "20752" uid: bed40e68-dadf-47b8-b904-47226e33c87a kind: Event lastTimestamp: "2026-07-02T21:41:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:41:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:09Z" name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt.18be96f23cf0afc7 namespace: kserve-ci-e2e-test resourceVersion: "20817" uid: 7f828abd-c6ab-4833-ac00-bfa69231907b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt namespace: kserve-ci-e2e-test resourceVersion: "20752" uid: bed40e68-dadf-47b8-b904-47226e33c87a kind: Event lastTimestamp: "2026-07-02T21:41:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:41:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:09Z" name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt.18be96f2401d6b26 namespace: kserve-ci-e2e-test resourceVersion: "20819" uid: 89d6b5df-d950-4d8f-8aca-acf185469fa0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt namespace: kserve-ci-e2e-test resourceVersion: "20752" uid: bed40e68-dadf-47b8-b904-47226e33c87a kind: Event lastTimestamp: "2026-07-02T21:41:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:41:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:41:09Z" name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt.18be96f24090c473 namespace: kserve-ci-e2e-test resourceVersion: "20820" uid: fc778e56-8e04-4ede-a345-b7fff03c76f5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T21:41:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt namespace: kserve-ci-e2e-test resourceVersion: "20752" uid: bed40e68-dadf-47b8-b904-47226e33c87a kind: Event lastTimestamp: "2026-07-02T21:42:37Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:41:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:37Z" name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt.18be96f2b1468787 namespace: kserve-ci-e2e-test resourceVersion: "21473" uid: b78a677c-3641-4a71-a708-c0cf74c1ba4b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt namespace: kserve-ci-e2e-test resourceVersion: "20752" uid: bed40e68-dadf-47b8-b904-47226e33c87a kind: Event lastTimestamp: "2026-07-02T21:42:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:42:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:34Z" name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt.18be970614f50724 namespace: kserve-ci-e2e-test resourceVersion: "21450" uid: b8397d28-a6a8-4a1e-bcab-2d3e3083e7e1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt namespace: kserve-ci-e2e-test resourceVersion: "20752" uid: bed40e68-dadf-47b8-b904-47226e33c87a kind: Event lastTimestamp: "2026-07-02T21:42:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:42:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:34Z" name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt.18be970614f5cfc9 namespace: kserve-ci-e2e-test resourceVersion: "21451" uid: fa60d8b1-ca30-427c-a4b9-f01ededb9f3d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt namespace: kserve-ci-e2e-test resourceVersion: "20752" uid: bed40e68-dadf-47b8-b904-47226e33c87a kind: Event lastTimestamp: "2026-07-02T21:42:37Z" message: 'Readiness probe failed: Get "https://10.132.0.34:8643/healthz": dial tcp 10.132.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:42:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:37Z" name: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt.18be9706b77e6e0f namespace: kserve-ci-e2e-test resourceVersion: "21472" uid: 6dcf439a-c599-45fa-af8f-5d0d1f7cbc04 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-efdfd-predictor-7fdc445bdf namespace: kserve-ci-e2e-test resourceVersion: "20738" uid: 690880d5-8826-4d3b-8eb0-9280e4e18a6e kind: Event lastTimestamp: "2026-07-02T21:41:04Z" message: 'Created pod: raw-sklearn-efdfd-predictor-7fdc445bdf-4z5bt' metadata: creationTimestamp: "2026-07-02T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:04Z" name: raw-sklearn-efdfd-predictor-7fdc445bdf.18be96f118a36f9a namespace: kserve-ci-e2e-test resourceVersion: "20751" uid: 3069618c-412b-48e8-a068-4f914b5d9873 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-efdfd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20735" uid: ba735883-1490-4a61-be1e-04b6098d73b3 kind: Event lastTimestamp: "2026-07-02T21:41:04Z" message: Scaled up replica set raw-sklearn-efdfd-predictor-7fdc445bdf from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:04Z" name: raw-sklearn-efdfd-predictor.18be96f11509a0b4 namespace: kserve-ci-e2e-test resourceVersion: "20739" uid: b232bcd6-d2b2-4404-9cbc-9b30c9fa42f5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:41:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-efdfd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20737" uid: 6b71d0ab-2de4-4fb7-8536-509a1db81aa2 kind: Event lastTimestamp: "2026-07-02T21:41:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:49Z" name: raw-sklearn-efdfd-predictor.18be96f493c0b5e0 namespace: kserve-ci-e2e-test resourceVersion: "21107" uid: 0637070c-58aa-4ca9-aea7-6d896420fee9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:41:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-efdfd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20737" uid: 6b71d0ab-2de4-4fb7-8536-509a1db81aa2 kind: Event lastTimestamp: "2026-07-02T21:41:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:41:49Z" name: raw-sklearn-efdfd-predictor.18be96f49424f94c namespace: kserve-ci-e2e-test resourceVersion: "21108" uid: a1ba5ad2-84dd-490c-ad49-415a2d0ee222 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:41:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-efdfd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20900" uid: 6b71d0ab-2de4-4fb7-8536-509a1db81aa2 kind: Event lastTimestamp: "2026-07-02T21:42:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:41:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:42:34Z" name: raw-sklearn-efdfd-predictor.18be96f8133148f3 namespace: kserve-ci-e2e-test resourceVersion: "21434" uid: ec28d66a-d61e-46bf-89f0-8f0814d860e6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T21:41:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-efdfd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20900" uid: 6b71d0ab-2de4-4fb7-8536-509a1db81aa2 kind: Event lastTimestamp: "2026-07-02T21:42:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:41:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:42:34Z" name: raw-sklearn-efdfd-predictor.18be96f813753e05 namespace: kserve-ci-e2e-test resourceVersion: "21435" uid: 1299dbef-f3b3-4eee-be8a-e389e065f213 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T21:41:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-efdfd namespace: kserve-ci-e2e-test resourceVersion: "20728" uid: bdf16404-fcba-48cd-9887-3f160782d5b4 kind: Event lastTimestamp: "2026-07-02T21:42:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:42:27Z" name: raw-sklearn-efdfd.18be96f1150e7407 namespace: kserve-ci-e2e-test resourceVersion: "21381" uid: 81191053-7a76-4c3f-aae2-48e56e3cb824 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:41:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-efdfd namespace: kserve-ci-e2e-test resourceVersion: "20728" uid: bdf16404-fcba-48cd-9887-3f160782d5b4 kind: Event lastTimestamp: "2026-07-02T21:41:04Z" message: 'Failed to update status for InferenceService "raw-sklearn-efdfd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-efdfd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:41:04Z" name: raw-sklearn-efdfd.18be96f115e20844 namespace: kserve-ci-e2e-test resourceVersion: "20764" uid: 49db79d4-b6af-4116-aad2-32579f0dfed8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:41:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-efdfd namespace: kserve-ci-e2e-test resourceVersion: "20731" uid: bdf16404-fcba-48cd-9887-3f160782d5b4 kind: Event lastTimestamp: "2026-07-02T21:41:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-efdfd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:41:04Z" name: raw-sklearn-efdfd.18be96f11c68349e namespace: kserve-ci-e2e-test resourceVersion: "20765" uid: f836dd56-26c2-4d76-98c5-2c3ed4991dcb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-efdfd namespace: kserve-ci-e2e-test resourceVersion: "21379" uid: bdf16404-fcba-48cd-9887-3f160782d5b4 kind: Event lastTimestamp: "2026-07-02T21:42:27Z" message: InferenceService [raw-sklearn-efdfd] is Ready metadata: creationTimestamp: "2026-07-02T21:42:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:42:27Z" name: raw-sklearn-efdfd.18be970468e72307 namespace: kserve-ci-e2e-test resourceVersion: "21380" uid: 266e30d2-dff2-4cf6-8633-6d4a7af70603 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T21:42:40.011223Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b namespace: kserve-ci-e2e-test resourceVersion: "21516" uid: 33beb7cc-d071-4ccb-8c5b-e63577ffb3c4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b to ip-10-0-135-227.ec2.internal metadata: creationTimestamp: "2026-07-02T21:42:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T21:42:40Z" name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b.18be970747d87fa7 namespace: kserve-ci-e2e-test resourceVersion: "21521" uid: 44a4e557-b375-4ae0-9b17-d034f9741f05 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-558f78ddf7-rgj7w source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:40Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: 33beb7cc-d071-4ccb-8c5b-e63577ffb3c4 kind: Event lastTimestamp: "2026-07-02T21:42:40Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-runtime-9d762-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T21:42:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:40Z" name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b.18be970756d8519e namespace: kserve-ci-e2e-test resourceVersion: "21533" uid: 1935fc71-958e-4284-9442-1375d085f769 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:41Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b namespace: kserve-ci-e2e-test resourceVersion: "21525" uid: 33beb7cc-d071-4ccb-8c5b-e63577ffb3c4 kind: Event lastTimestamp: "2026-07-02T21:42:41Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T21:42:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T21:42:41Z" name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b.18be9707858dae71 namespace: kserve-ci-e2e-test resourceVersion: "21541" uid: 0c519190-cb01-4a30-b133-40a54a62434b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: 33beb7cc-d071-4ccb-8c5b-e63577ffb3c4 kind: Event lastTimestamp: "2026-07-02T21:42:41Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:33f82c067d520fe749f73749cf76b5935fe69bd9eb141504c6660698c7947743" already present on machine metadata: creationTimestamp: "2026-07-02T21:42:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:41Z" name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b.18be97078706e7f4 namespace: kserve-ci-e2e-test resourceVersion: "21544" uid: 07147e36-b0cf-49aa-8406-8fa57afebf47 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: 33beb7cc-d071-4ccb-8c5b-e63577ffb3c4 kind: Event lastTimestamp: "2026-07-02T21:42:41Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T21:42:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:41Z" name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b.18be9707898e9c22 namespace: kserve-ci-e2e-test resourceVersion: "21545" uid: e18d1521-fe76-40be-8b8c-93183c20f483 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: 33beb7cc-d071-4ccb-8c5b-e63577ffb3c4 kind: Event lastTimestamp: "2026-07-02T21:42:41Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T21:42:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:41Z" name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b.18be97078a0190fb namespace: kserve-ci-e2e-test resourceVersion: "21546" uid: 6b8c4908-fd2d-4296-9617-2896f952518d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: 33beb7cc-d071-4ccb-8c5b-e63577ffb3c4 kind: Event lastTimestamp: "2026-07-02T21:42:44Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1699" already present on machine metadata: creationTimestamp: "2026-07-02T21:42:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:44Z" name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b.18be97086fde55b2 namespace: kserve-ci-e2e-test resourceVersion: "21576" uid: cbb0c069-1152-48d2-b49a-6b2dc21e53de reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: 33beb7cc-d071-4ccb-8c5b-e63577ffb3c4 kind: Event lastTimestamp: "2026-07-02T21:42:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T21:42:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:45Z" name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b.18be970873260ac6 namespace: kserve-ci-e2e-test resourceVersion: "21580" uid: 76cd9b22-1065-453a-abb2-464e903a472e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: 33beb7cc-d071-4ccb-8c5b-e63577ffb3c4 kind: Event lastTimestamp: "2026-07-02T21:42:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T21:42:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:45Z" name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b.18be970873995fa5 namespace: kserve-ci-e2e-test resourceVersion: "21581" uid: 3da6c6f1-0807-43ad-be5e-d192eaa783e9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: 33beb7cc-d071-4ccb-8c5b-e63577ffb3c4 kind: Event lastTimestamp: "2026-07-02T21:42:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T21:42:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:45Z" name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b.18be970873a36c1c namespace: kserve-ci-e2e-test resourceVersion: "21582" uid: 8444efd7-abef-4a69-bda4-6044dc9120e0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: 33beb7cc-d071-4ccb-8c5b-e63577ffb3c4 kind: Event lastTimestamp: "2026-07-02T21:42:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T21:42:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:45Z" name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b.18be970876e048ef namespace: kserve-ci-e2e-test resourceVersion: "21585" uid: 1344fd5f-dc29-4136-a4d5-ab8aa1948a21 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: 33beb7cc-d071-4ccb-8c5b-e63577ffb3c4 kind: Event lastTimestamp: "2026-07-02T21:42:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:42:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:42:45Z" name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b.18be970877549938 namespace: kserve-ci-e2e-test resourceVersion: "21586" uid: e27d19ab-49be-445a-aba5-c4f42fef421d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T21:42:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: 33beb7cc-d071-4ccb-8c5b-e63577ffb3c4 kind: Event lastTimestamp: "2026-07-02T21:43:52Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:42:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:43:52Z" name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b.18be9708e7b5000d namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: d461c81b-b9e0-4abe-bc86-704d1dbc39f6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:44:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: 33beb7cc-d071-4ccb-8c5b-e63577ffb3c4 kind: Event lastTimestamp: "2026-07-02T21:44:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T21:44:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:44:00Z" name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b.18be9719f7843d23 namespace: kserve-ci-e2e-test resourceVersion: "22142" uid: ad17382c-6d2e-441a-9488-5eb03ed303b8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:44:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: 33beb7cc-d071-4ccb-8c5b-e63577ffb3c4 kind: Event lastTimestamp: "2026-07-02T21:44:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T21:44:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:44:00Z" name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b.18be9719f784c96b namespace: kserve-ci-e2e-test resourceVersion: "22143" uid: 5669812d-d721-4683-86c1-4374f6481110 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:44:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: 33beb7cc-d071-4ccb-8c5b-e63577ffb3c4 kind: Event lastTimestamp: "2026-07-02T21:44:02Z" message: 'Readiness probe failed: Get "https://10.132.0.35:8643/healthz": dial tcp 10.132.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T21:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T21:44:02Z" name: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b.18be971a99d73a0c namespace: kserve-ci-e2e-test resourceVersion: "22165" uid: e9835850-026a-4e82-8832-add39636d3ef reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-227.ec2.internal source: component: kubelet host: ip-10-0-135-227.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-9d762-predictor-76c5cffb77 namespace: kserve-ci-e2e-test resourceVersion: "21502" uid: e4679849-a223-4de0-93fe-99c29b30f33e kind: Event lastTimestamp: "2026-07-02T21:42:39Z" message: 'Created pod: raw-sklearn-runtime-9d762-predictor-76c5cffb77-6968b' metadata: creationTimestamp: "2026-07-02T21:42:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:42:40Z" name: raw-sklearn-runtime-9d762-predictor-76c5cffb77.18be9707472181ba namespace: kserve-ci-e2e-test resourceVersion: "21517" uid: 5355a466-6fc7-4103-ae32-1d84645676cd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-9d762-predictor namespace: kserve-ci-e2e-test resourceVersion: "21500" uid: 81b7ff7b-e36b-4418-a424-950d3dce9bb0 kind: Event lastTimestamp: "2026-07-02T21:42:39Z" message: Scaled up replica set raw-sklearn-runtime-9d762-predictor-76c5cffb77 from 0 to 1 metadata: creationTimestamp: "2026-07-02T21:42:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:42:39Z" name: raw-sklearn-runtime-9d762-predictor.18be97074295e98e namespace: kserve-ci-e2e-test resourceVersion: "21504" uid: d7001aae-57c5-43a8-8109-7de50827c637 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:42:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-9d762-predictor namespace: kserve-ci-e2e-test resourceVersion: "21501" uid: c96540fc-8daa-4311-87a5-4c025692accd kind: Event lastTimestamp: "2026-07-02T21:43:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:42:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:43:09Z" name: raw-sklearn-runtime-9d762-predictor.18be970ac127b0b4 namespace: kserve-ci-e2e-test resourceVersion: "21763" uid: 3ae8e284-a8bc-4ce9-b701-e2d05349bb92 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:42:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-9d762-predictor namespace: kserve-ci-e2e-test resourceVersion: "21501" uid: c96540fc-8daa-4311-87a5-4c025692accd kind: Event lastTimestamp: "2026-07-02T21:43:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T21:42:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:43:09Z" name: raw-sklearn-runtime-9d762-predictor.18be970ac19e3cb7 namespace: kserve-ci-e2e-test resourceVersion: "21764" uid: e0d357cf-eb0f-434b-8a05-f5902ea86545 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:43:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-9d762-predictor namespace: kserve-ci-e2e-test resourceVersion: "21660" uid: c96540fc-8daa-4311-87a5-4c025692accd kind: Event lastTimestamp: "2026-07-02T21:43:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:43:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:43:55Z" name: raw-sklearn-runtime-9d762-predictor.18be9711bfb7d6f4 namespace: kserve-ci-e2e-test resourceVersion: "22082" uid: ab6009d3-2d8b-4890-bc57-24ab76c06eb5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T21:43:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-9d762-predictor namespace: kserve-ci-e2e-test resourceVersion: "21660" uid: c96540fc-8daa-4311-87a5-4c025692accd kind: Event lastTimestamp: "2026-07-02T21:43:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T21:43:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T21:43:55Z" name: raw-sklearn-runtime-9d762-predictor.18be9711c020e159 namespace: kserve-ci-e2e-test resourceVersion: "22083" uid: bbf9fab5-bc1d-47e4-996e-b5bdb55dad25 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-02T21:42:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-9d762 namespace: kserve-ci-e2e-test resourceVersion: "21493" uid: 7f02d09d-bf92-4f24-8ac7-08fb3f34974f kind: Event lastTimestamp: "2026-07-02T21:43:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T21:42:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:43:59Z" name: raw-sklearn-runtime-9d762.18be970742994e9e namespace: kserve-ci-e2e-test resourceVersion: "22118" uid: ab933bdf-0f5a-43d8-b803-d60c6e6a79b8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T21:42:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-9d762 namespace: kserve-ci-e2e-test resourceVersion: "21493" uid: 7f02d09d-bf92-4f24-8ac7-08fb3f34974f kind: Event lastTimestamp: "2026-07-02T21:42:40Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-9d762": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-9d762": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:42:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:42:40Z" name: raw-sklearn-runtime-9d762.18be9707433a26b5 namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: 90db3b8c-d8c0-4603-8274-2a9f169ff1a2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:42:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-9d762 namespace: kserve-ci-e2e-test resourceVersion: "21497" uid: 7f02d09d-bf92-4f24-8ac7-08fb3f34974f kind: Event lastTimestamp: "2026-07-02T21:42:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-9d762": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T21:42:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:42:40Z" name: raw-sklearn-runtime-9d762.18be970749d6e37c namespace: kserve-ci-e2e-test resourceVersion: "21529" uid: 09f62c8d-f308-4a17-8249-f717aeca8f65 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T21:43:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-9d762 namespace: kserve-ci-e2e-test resourceVersion: "22115" uid: 7f02d09d-bf92-4f24-8ac7-08fb3f34974f kind: Event lastTimestamp: "2026-07-02T21:43:59Z" message: InferenceService [raw-sklearn-runtime-9d762] is Ready metadata: creationTimestamp: "2026-07-02T21:43:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T21:43:59Z" name: raw-sklearn-runtime-9d762.18be9719b27f37b8 namespace: kserve-ci-e2e-test resourceVersion: "22116" uid: 7f6b4f36-1884-45ec-9696-d25382939503 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "22397"