--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:00:38.497130Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-40cfe0-predictor-5b45d7457b-xfdx7 namespace: kserve-ci-e2e-test resourceVersion: "20062" uid: 868ea921-2bf6-4c66-a488-80511d4c1b8a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-40cfe0-predictor-5b45d7457b-xfdx7 to ip-10-0-134-57.ec2.internal metadata: creationTimestamp: "2026-07-10T19:00:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:00:38Z" name: isvc-init-fail-40cfe0-predictor-5b45d7457b-xfdx7.18c102d45aeba745 namespace: kserve-ci-e2e-test resourceVersion: "20066" uid: 718ae195-d199-449a-b511-95a90eef74a4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-856b4cf847-czcjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:38Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-40cfe0-predictor-5b45d7457b-xfdx7 namespace: kserve-ci-e2e-test resourceVersion: "20064" uid: 868ea921-2bf6-4c66-a488-80511d4c1b8a kind: Event lastTimestamp: "2026-07-10T19:00:38Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-init-fail-40cfe0-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:00:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:00:38Z" name: isvc-init-fail-40cfe0-predictor-5b45d7457b-xfdx7.18c102d46ae759fd namespace: kserve-ci-e2e-test resourceVersion: "20078" uid: bac87305-8ef4-4e94-ae47-3465e5834866 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-40cfe0-predictor-5b45d7457b-xfdx7 namespace: kserve-ci-e2e-test resourceVersion: "20070" uid: 868ea921-2bf6-4c66-a488-80511d4c1b8a kind: Event lastTimestamp: "2026-07-10T19:00:39Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:00:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:00:39Z" name: isvc-init-fail-40cfe0-predictor-5b45d7457b-xfdx7.18c102d4989fc0ea namespace: kserve-ci-e2e-test resourceVersion: "20092" uid: 324d3695-2b4d-4e12-8831-15a45e626bfb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:00:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-40cfe0-predictor-5b45d7457b-xfdx7 namespace: kserve-ci-e2e-test resourceVersion: "20064" uid: 868ea921-2bf6-4c66-a488-80511d4c1b8a kind: Event lastTimestamp: "2026-07-10T19:00:43Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:e7d18b8520466922c73925c5601004b31b3c3ec90f89a0821d13bdeee1848812" already present on machine metadata: creationTimestamp: "2026-07-10T19:00:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:00:43Z" name: isvc-init-fail-40cfe0-predictor-5b45d7457b-xfdx7.18c102d49a1f79d2 namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: 137b21cb-d02b-40ef-88f7-8245c9bc85d3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:00:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-40cfe0-predictor-5b45d7457b-xfdx7 namespace: kserve-ci-e2e-test resourceVersion: "20064" uid: 868ea921-2bf6-4c66-a488-80511d4c1b8a kind: Event lastTimestamp: "2026-07-10T19:00:43Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:00:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:00:43Z" name: isvc-init-fail-40cfe0-predictor-5b45d7457b-xfdx7.18c102d49d1e535d namespace: kserve-ci-e2e-test resourceVersion: "20138" uid: 7426baca-2c2e-4470-ad2b-139306464ddc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:00:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-40cfe0-predictor-5b45d7457b-xfdx7 namespace: kserve-ci-e2e-test resourceVersion: "20064" uid: 868ea921-2bf6-4c66-a488-80511d4c1b8a kind: Event lastTimestamp: "2026-07-10T19:00:44Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:00:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:00:44Z" name: isvc-init-fail-40cfe0-predictor-5b45d7457b-xfdx7.18c102d49d9b46ff namespace: kserve-ci-e2e-test resourceVersion: "20139" uid: 60cb86a0-5b98-462b-be33-16f88c582a41 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-40cfe0-predictor-5b45d7457b-xfdx7 namespace: kserve-ci-e2e-test resourceVersion: "20064" uid: 868ea921-2bf6-4c66-a488-80511d4c1b8a kind: Event lastTimestamp: "2026-07-10T19:00:47Z" message: Back-off restarting failed container storage-initializer in pod isvc-init-fail-40cfe0-predictor-5b45d7457b-xfdx7_kserve-ci-e2e-test(868ea921-2bf6-4c66-a488-80511d4c1b8a) metadata: creationTimestamp: "2026-07-10T19:00:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:00:47Z" name: isvc-init-fail-40cfe0-predictor-5b45d7457b-xfdx7.18c102d68fa9a159 namespace: kserve-ci-e2e-test resourceVersion: "20172" uid: 0ef0cc62-ce27-4f3d-b9d4-fc1ed2a61c8f reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-40cfe0-predictor-5b45d7457b namespace: kserve-ci-e2e-test resourceVersion: "20052" uid: b89f08ea-0f2a-4cce-815d-a902b4616fc8 kind: Event lastTimestamp: "2026-07-10T19:00:38Z" message: 'Created pod: isvc-init-fail-40cfe0-predictor-5b45d7457b-xfdx7' metadata: creationTimestamp: "2026-07-10T19:00:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:00:38Z" name: isvc-init-fail-40cfe0-predictor-5b45d7457b.18c102d45a13dfc2 namespace: kserve-ci-e2e-test resourceVersion: "20063" uid: 4094d5ca-9ad2-45d0-9444-c76455320fc1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-40cfe0-predictor namespace: kserve-ci-e2e-test resourceVersion: "20049" uid: e157abd0-0e8a-4862-a715-b4121964f4f0 kind: Event lastTimestamp: "2026-07-10T19:00:38Z" message: Scaled up replica set isvc-init-fail-40cfe0-predictor-5b45d7457b from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:00:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:00:38Z" name: isvc-init-fail-40cfe0-predictor.18c102d4564ef7a3 namespace: kserve-ci-e2e-test resourceVersion: "20055" uid: 0d0e7044-3c85-463c-a589-22246d37d2dc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-10T19:00:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-40cfe0 namespace: kserve-ci-e2e-test resourceVersion: "20024" uid: 0dc55dfa-5841-428f-acb9-94473b763cf8 kind: Event lastTimestamp: "2026-07-10T19:00:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:00:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:00:48Z" name: isvc-init-fail-40cfe0.18c102d45612d09a namespace: kserve-ci-e2e-test resourceVersion: "20177" uid: 30cd0980-1eb0-4194-a3c0-35d221b920f8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:00:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-40cfe0 namespace: kserve-ci-e2e-test resourceVersion: "20024" uid: 0dc55dfa-5841-428f-acb9-94473b763cf8 kind: Event lastTimestamp: "2026-07-10T19:00:38Z" message: 'Failed to update status for InferenceService "isvc-init-fail-40cfe0": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-40cfe0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:00:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:00:38Z" name: isvc-init-fail-40cfe0.18c102d4567b9607 namespace: kserve-ci-e2e-test resourceVersion: "20072" uid: abcfb666-131c-4572-b03f-6223c2993b73 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:00:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-40cfe0 namespace: kserve-ci-e2e-test resourceVersion: "20024" uid: 0dc55dfa-5841-428f-acb9-94473b763cf8 kind: Event lastTimestamp: "2026-07-10T19:00:38Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-40cfe0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:00:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:00:38Z" name: isvc-init-fail-40cfe0.18c102d4567c002d namespace: kserve-ci-e2e-test resourceVersion: "20073" uid: e61ac171-08d8-4bd5-8f3e-91cec5c337a7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-10T18:54:58.519992Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt namespace: kserve-ci-e2e-test resourceVersion: "17269" uid: 2c2dc777-0b48-4019-9589-ad4bde6e3142 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt to ip-10-0-134-57.ec2.internal metadata: creationTimestamp: "2026-07-10T18:54:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T18:54:58Z" name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt.18c1028532b43b4f namespace: kserve-ci-e2e-test resourceVersion: "17275" uid: f4d10996-0048-4c52-98e3-94828e867205 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-856b4cf847-czcjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:58Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: 2c2dc777-0b48-4019-9589-ad4bde6e3142 kind: Event lastTimestamp: "2026-07-10T18:54:58Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-logger-raw-bb4f7-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T18:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:54:58Z" name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt.18c1028541c2ed5c namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 711a014c-bf3e-49ab-a8e9-c153ec0ae817 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt namespace: kserve-ci-e2e-test resourceVersion: "17279" uid: 2c2dc777-0b48-4019-9589-ad4bde6e3142 kind: Event lastTimestamp: "2026-07-10T18:54:59Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T18:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T18:54:59Z" name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt.18c10285709c6e8f namespace: kserve-ci-e2e-test resourceVersion: "17293" uid: 14f29691-1092-4bc2-ab9d-42be1016bfd9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: 2c2dc777-0b48-4019-9589-ad4bde6e3142 kind: Event lastTimestamp: "2026-07-10T18:54:59Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:e7d18b8520466922c73925c5601004b31b3c3ec90f89a0821d13bdeee1848812" already present on machine metadata: creationTimestamp: "2026-07-10T18:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:54:59Z" name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt.18c1028572656197 namespace: kserve-ci-e2e-test resourceVersion: "17295" uid: 6cbd4c10-f266-47cf-9e77-6329323841b9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: 2c2dc777-0b48-4019-9589-ad4bde6e3142 kind: Event lastTimestamp: "2026-07-10T18:54:59Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T18:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:54:59Z" name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt.18c10285762ef952 namespace: kserve-ci-e2e-test resourceVersion: "17296" uid: b9e3626a-a55f-41a2-a6af-e7af11fed7e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: 2c2dc777-0b48-4019-9589-ad4bde6e3142 kind: Event lastTimestamp: "2026-07-10T18:54:59Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T18:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:54:59Z" name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt.18c1028576ad3398 namespace: kserve-ci-e2e-test resourceVersion: "17297" uid: 8ce4191a-6791-493e-86c0-b13004b942fb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: 2c2dc777-0b48-4019-9589-ad4bde6e3142 kind: Event lastTimestamp: "2026-07-10T18:55:04Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1724" already present on machine metadata: creationTimestamp: "2026-07-10T18:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:55:04Z" name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt.18c102869335551d namespace: kserve-ci-e2e-test resourceVersion: "17339" uid: b6b436fd-1d85-4a8f-a2aa-943acfecf9f2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: 2c2dc777-0b48-4019-9589-ad4bde6e3142 kind: Event lastTimestamp: "2026-07-10T18:55:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T18:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:55:04Z" name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt.18c10286961c35e5 namespace: kserve-ci-e2e-test resourceVersion: "17341" uid: 2afd7139-affd-4522-9b9c-9fcb909da916 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: 2c2dc777-0b48-4019-9589-ad4bde6e3142 kind: Event lastTimestamp: "2026-07-10T18:55:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T18:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:55:04Z" name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt.18c10286969f9d80 namespace: kserve-ci-e2e-test resourceVersion: "17342" uid: 73fb58d3-6250-4117-896c-2ed52dcfb854 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: 2c2dc777-0b48-4019-9589-ad4bde6e3142 kind: Event lastTimestamp: "2026-07-10T18:55:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T18:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:55:04Z" name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt.18c1028696ad0826 namespace: kserve-ci-e2e-test resourceVersion: "17345" uid: 8041b9db-8e8e-4f3a-a872-981c6ceb260d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: 2c2dc777-0b48-4019-9589-ad4bde6e3142 kind: Event lastTimestamp: "2026-07-10T18:55:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T18:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:55:04Z" name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt.18c1028699bc3f1f namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: d529cb8a-e89f-4e7f-b0fd-0eecd42313d6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: 2c2dc777-0b48-4019-9589-ad4bde6e3142 kind: Event lastTimestamp: "2026-07-10T18:55:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T18:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:55:04Z" name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt.18c102869a3bb3c7 namespace: kserve-ci-e2e-test resourceVersion: "17348" uid: 02b655ce-2dc7-4410-b900-55cf923319ae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: 2c2dc777-0b48-4019-9589-ad4bde6e3142 kind: Event lastTimestamp: "2026-07-10T18:55:04Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:f89d75505dfa0eae68aea417576d9edc02c9c812117fbdf23c2a565323101ab5" already present on machine metadata: creationTimestamp: "2026-07-10T18:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:55:04Z" name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt.18c102869a472c26 namespace: kserve-ci-e2e-test resourceVersion: "17349" uid: f650394a-21ef-4a9c-8bf7-fc07c2ceeb1b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: 2c2dc777-0b48-4019-9589-ad4bde6e3142 kind: Event lastTimestamp: "2026-07-10T18:55:04Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-10T18:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:55:04Z" name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt.18c102869d3485f9 namespace: kserve-ci-e2e-test resourceVersion: "17350" uid: d13d8f3e-df64-49b9-b991-1d24214eefcd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: 2c2dc777-0b48-4019-9589-ad4bde6e3142 kind: Event lastTimestamp: "2026-07-10T18:55:04Z" message: Started container agent metadata: creationTimestamp: "2026-07-10T18:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:55:04Z" name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt.18c102869db5183f namespace: kserve-ci-e2e-test resourceVersion: "17351" uid: 4410ab16-72ea-4900-a399-c6b0d111a4d8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-10T18:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: 2c2dc777-0b48-4019-9589-ad4bde6e3142 kind: Event lastTimestamp: "2026-07-10T18:56:41Z" message: 'Readiness probe failed: dial tcp 10.132.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T18:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:56:41Z" name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt.18c10286cf45a473 namespace: kserve-ci-e2e-test resourceVersion: "18172" uid: 0edad218-a305-4b63-a27a-92bb028e1934 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-10T18:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: 2c2dc777-0b48-4019-9589-ad4bde6e3142 kind: Event lastTimestamp: "2026-07-10T18:56:41Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-10T18:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:56:41Z" name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt.18c10286cf518346 namespace: kserve-ci-e2e-test resourceVersion: "18173" uid: 96f6e6b0-a376-454a-902e-df24c2ef2f64 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:56:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: 2c2dc777-0b48-4019-9589-ad4bde6e3142 kind: Event lastTimestamp: "2026-07-10T18:56:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T18:56:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:56:23Z" name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt.18c10299092fa6f9 namespace: kserve-ci-e2e-test resourceVersion: "17972" uid: 710314d7-5f65-4b17-9f4a-b77511ed9452 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:56:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: 2c2dc777-0b48-4019-9589-ad4bde6e3142 kind: Event lastTimestamp: "2026-07-10T18:56:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T18:56:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:56:23Z" name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt.18c102990930ef55 namespace: kserve-ci-e2e-test resourceVersion: "17975" uid: 2ea9613a-4733-456a-94ea-2f426c49c6f4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:56:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: 2c2dc777-0b48-4019-9589-ad4bde6e3142 kind: Event lastTimestamp: "2026-07-10T18:56:23Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-10T18:56:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:56:23Z" name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt.18c10299094055e7 namespace: kserve-ci-e2e-test resourceVersion: "17976" uid: 61f0c59e-cc16-4055-92c5-7a87014df724 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T18:56:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: 2c2dc777-0b48-4019-9589-ad4bde6e3142 kind: Event lastTimestamp: "2026-07-10T18:56:41Z" message: 'Readiness probe failed: Get "https://10.132.0.29:8643/healthz": dial tcp 10.132.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T18:56:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:56:41Z" name: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt.18c10299ab6b7cef namespace: kserve-ci-e2e-test resourceVersion: "18171" uid: f22f83b3-39e0-4806-a064-4c418b40b03a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-bb4f7-predictor-78c88b68dd namespace: kserve-ci-e2e-test resourceVersion: "17260" uid: 94f89593-5cbf-4038-bca3-7e2ec40a9d70 kind: Event lastTimestamp: "2026-07-10T18:54:58Z" message: 'Created pod: isvc-logger-raw-bb4f7-predictor-78c88b68dd-5wcbt' metadata: creationTimestamp: "2026-07-10T18:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:54:58Z" name: isvc-logger-raw-bb4f7-predictor-78c88b68dd.18c1028531fc0891 namespace: kserve-ci-e2e-test resourceVersion: "17273" uid: 48e2f2ab-b2bc-4c66-812f-0e92bcdc2eeb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-bb4f7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17258" uid: 4e73edf6-0229-4e9a-94d5-901393779e11 kind: Event lastTimestamp: "2026-07-10T18:54:58Z" message: Scaled up replica set isvc-logger-raw-bb4f7-predictor-78c88b68dd from 0 to 1 metadata: creationTimestamp: "2026-07-10T18:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:54:58Z" name: isvc-logger-raw-bb4f7-predictor.18c102852ee1101e namespace: kserve-ci-e2e-test resourceVersion: "17263" uid: ef072cb6-eb58-404c-ada1-46eb2b198406 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T18:55:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-bb4f7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17259" uid: f2c7b6d7-7b9d-4f7a-8b19-59968ff45887 kind: Event lastTimestamp: "2026-07-10T18:55:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T18:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:55:43Z" name: isvc-logger-raw-bb4f7-predictor.18c10288ad6fe61a namespace: kserve-ci-e2e-test resourceVersion: "17649" uid: a312f655-ff4b-4b6b-a5e4-16706809387b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T18:55:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-bb4f7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17259" uid: f2c7b6d7-7b9d-4f7a-8b19-59968ff45887 kind: Event lastTimestamp: "2026-07-10T18:55:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T18:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:55:43Z" name: isvc-logger-raw-bb4f7-predictor.18c10288ade2ff55 namespace: kserve-ci-e2e-test resourceVersion: "17650" uid: 2c4f67c8-2185-4bb2-9bf2-d2d288c2bb39 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T18:55:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-bb4f7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17433" uid: f2c7b6d7-7b9d-4f7a-8b19-59968ff45887 kind: Event lastTimestamp: "2026-07-10T18:56:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T18:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:56:13Z" name: isvc-logger-raw-bb4f7-predictor.18c1028c2ca0198d namespace: kserve-ci-e2e-test resourceVersion: "17871" uid: 030e78b7-905b-40eb-b468-e16d67f3f77e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T18:55:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-bb4f7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17433" uid: f2c7b6d7-7b9d-4f7a-8b19-59968ff45887 kind: Event lastTimestamp: "2026-07-10T18:56:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T18:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:56:13Z" name: isvc-logger-raw-bb4f7-predictor.18c1028c2d03bb45 namespace: kserve-ci-e2e-test resourceVersion: "17872" uid: fc4107bb-1f19-4a5a-8696-26f62b588fbe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-10T18:54:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-bb4f7 namespace: kserve-ci-e2e-test resourceVersion: "17251" uid: cec1e6cd-c611-4a45-aa5f-4bdb56ae8def kind: Event lastTimestamp: "2026-07-10T18:56:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T18:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:56:11Z" name: isvc-logger-raw-bb4f7.18c102852eaca2ce namespace: kserve-ci-e2e-test resourceVersion: "17857" uid: 0c8261d3-0013-40b2-9bcc-cd7d199a672a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-bb4f7 namespace: kserve-ci-e2e-test resourceVersion: "17251" uid: cec1e6cd-c611-4a45-aa5f-4bdb56ae8def kind: Event lastTimestamp: "2026-07-10T18:54:58Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-bb4f7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-bb4f7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T18:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:54:58Z" name: isvc-logger-raw-bb4f7.18c102852f3ed1bf namespace: kserve-ci-e2e-test resourceVersion: "17265" uid: 84df54a5-f4b3-45d9-8e63-2e2ead014fed reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:56:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-bb4f7 namespace: kserve-ci-e2e-test resourceVersion: "17855" uid: cec1e6cd-c611-4a45-aa5f-4bdb56ae8def kind: Event lastTimestamp: "2026-07-10T18:56:11Z" message: InferenceService [isvc-logger-raw-bb4f7] is Ready metadata: creationTimestamp: "2026-07-10T18:56:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:56:11Z" name: isvc-logger-raw-bb4f7.18c1029635184306 namespace: kserve-ci-e2e-test resourceVersion: "17856" uid: 38b86ced-2174-49be-bec4-6afaf600a593 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T18:59:04.160038Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-37e3de-predictor-8557c96d5-d7527 namespace: kserve-ci-e2e-test resourceVersion: "19220" uid: 634a823b-50a1-443f-862b-e5afaa378932 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-37e3de-predictor-8557c96d5-d7527 to ip-10-0-134-57.ec2.internal metadata: creationTimestamp: "2026-07-10T18:59:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T18:59:04Z" name: isvc-primary-37e3de-predictor-8557c96d5-d7527.18c102be63fddf74 namespace: kserve-ci-e2e-test resourceVersion: "19227" uid: 0ce73f86-ae81-44ce-9e33-9031a15a2800 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-856b4cf847-czcjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:59:04Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-37e3de-predictor-8557c96d5-d7527 namespace: kserve-ci-e2e-test resourceVersion: "19233" uid: 634a823b-50a1-443f-862b-e5afaa378932 kind: Event lastTimestamp: "2026-07-10T18:59:04Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T18:59:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T18:59:04Z" name: isvc-primary-37e3de-predictor-8557c96d5-d7527.18c102be7ec46656 namespace: kserve-ci-e2e-test resourceVersion: "19242" uid: d8bbad15-89cb-4754-bc03-febc10bf2fd5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:59:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-37e3de-predictor-8557c96d5-d7527 namespace: kserve-ci-e2e-test resourceVersion: "19223" uid: 634a823b-50a1-443f-862b-e5afaa378932 kind: Event lastTimestamp: "2026-07-10T18:59:04Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:e7d18b8520466922c73925c5601004b31b3c3ec90f89a0821d13bdeee1848812" already present on machine metadata: creationTimestamp: "2026-07-10T18:59:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:59:04Z" name: isvc-primary-37e3de-predictor-8557c96d5-d7527.18c102be80602b32 namespace: kserve-ci-e2e-test resourceVersion: "19244" uid: 72d1ab25-827f-4c1f-baa1-7007c9422706 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:59:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-37e3de-predictor-8557c96d5-d7527 namespace: kserve-ci-e2e-test resourceVersion: "19223" uid: 634a823b-50a1-443f-862b-e5afaa378932 kind: Event lastTimestamp: "2026-07-10T18:59:04Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T18:59:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:59:04Z" name: isvc-primary-37e3de-predictor-8557c96d5-d7527.18c102be82fc5c0b namespace: kserve-ci-e2e-test resourceVersion: "19246" uid: 5f4fe9ea-4fdf-4b05-b330-f22077e8de91 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:59:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-37e3de-predictor-8557c96d5-d7527 namespace: kserve-ci-e2e-test resourceVersion: "19223" uid: 634a823b-50a1-443f-862b-e5afaa378932 kind: Event lastTimestamp: "2026-07-10T18:59:04Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T18:59:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:59:04Z" name: isvc-primary-37e3de-predictor-8557c96d5-d7527.18c102be8394f703 namespace: kserve-ci-e2e-test resourceVersion: "19248" uid: c9508cc3-8694-4cf0-8de5-5565375feaa2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:59:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-37e3de-predictor-8557c96d5-d7527 namespace: kserve-ci-e2e-test resourceVersion: "19223" uid: 634a823b-50a1-443f-862b-e5afaa378932 kind: Event lastTimestamp: "2026-07-10T18:59:09Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1724" already present on machine metadata: creationTimestamp: "2026-07-10T18:59:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:59:09Z" name: isvc-primary-37e3de-predictor-8557c96d5-d7527.18c102bfa2620fca namespace: kserve-ci-e2e-test resourceVersion: "19293" uid: 5128e7c0-8273-49ce-a459-760807dd6643 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:59:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-37e3de-predictor-8557c96d5-d7527 namespace: kserve-ci-e2e-test resourceVersion: "19223" uid: 634a823b-50a1-443f-862b-e5afaa378932 kind: Event lastTimestamp: "2026-07-10T18:59:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T18:59:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:59:09Z" name: isvc-primary-37e3de-predictor-8557c96d5-d7527.18c102bfa5f18b09 namespace: kserve-ci-e2e-test resourceVersion: "19295" uid: 266318ec-42ed-4e6d-b4d0-519668d67a7a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:59:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-37e3de-predictor-8557c96d5-d7527 namespace: kserve-ci-e2e-test resourceVersion: "19223" uid: 634a823b-50a1-443f-862b-e5afaa378932 kind: Event lastTimestamp: "2026-07-10T18:59:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T18:59:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:59:09Z" name: isvc-primary-37e3de-predictor-8557c96d5-d7527.18c102bfa6766ee3 namespace: kserve-ci-e2e-test resourceVersion: "19297" uid: 1bb20335-627c-45ba-9e1d-d7cef54e8fb1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:59:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-37e3de-predictor-8557c96d5-d7527 namespace: kserve-ci-e2e-test resourceVersion: "19223" uid: 634a823b-50a1-443f-862b-e5afaa378932 kind: Event lastTimestamp: "2026-07-10T18:59:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T18:59:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:59:09Z" name: isvc-primary-37e3de-predictor-8557c96d5-d7527.18c102bfa682ea2e namespace: kserve-ci-e2e-test resourceVersion: "19299" uid: 9ad03dfe-3c77-46cd-adfa-267d21a6610d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:59:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-37e3de-predictor-8557c96d5-d7527 namespace: kserve-ci-e2e-test resourceVersion: "19223" uid: 634a823b-50a1-443f-862b-e5afaa378932 kind: Event lastTimestamp: "2026-07-10T18:59:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T18:59:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:59:09Z" name: isvc-primary-37e3de-predictor-8557c96d5-d7527.18c102bfa9bacc1f namespace: kserve-ci-e2e-test resourceVersion: "19300" uid: 89989e8a-5770-4a04-a1b6-e7d16f67d0ec reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:59:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-37e3de-predictor-8557c96d5-d7527 namespace: kserve-ci-e2e-test resourceVersion: "19223" uid: 634a823b-50a1-443f-862b-e5afaa378932 kind: Event lastTimestamp: "2026-07-10T18:59:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T18:59:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:59:09Z" name: isvc-primary-37e3de-predictor-8557c96d5-d7527.18c102bfaa3b3b59 namespace: kserve-ci-e2e-test resourceVersion: "19302" uid: 799ef9f9-e6f1-458b-a726-1cb70461933f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-10T18:59:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-37e3de-predictor-8557c96d5-d7527 namespace: kserve-ci-e2e-test resourceVersion: "19223" uid: 634a823b-50a1-443f-862b-e5afaa378932 kind: Event lastTimestamp: "2026-07-10T19:00:06Z" message: 'Readiness probe failed: dial tcp 10.132.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T18:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:00:06Z" name: isvc-primary-37e3de-predictor-8557c96d5-d7527.18c102bfde7b3e9e namespace: kserve-ci-e2e-test resourceVersion: "19701" uid: e3e62294-bf7d-4f39-a1ed-abc838dc1267 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-37e3de-predictor-8557c96d5-d7527 namespace: kserve-ci-e2e-test resourceVersion: "19223" uid: 634a823b-50a1-443f-862b-e5afaa378932 kind: Event lastTimestamp: "2026-07-10T19:00:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:00:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:00:38Z" name: isvc-primary-37e3de-predictor-8557c96d5-d7527.18c102d450c4ac68 namespace: kserve-ci-e2e-test resourceVersion: "20025" uid: e4fe05c8-cd94-4617-a99e-6a600ee8205d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-37e3de-predictor-8557c96d5-d7527 namespace: kserve-ci-e2e-test resourceVersion: "19223" uid: 634a823b-50a1-443f-862b-e5afaa378932 kind: Event lastTimestamp: "2026-07-10T19:00:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:00:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:00:38Z" name: isvc-primary-37e3de-predictor-8557c96d5-d7527.18c102d450c54217 namespace: kserve-ci-e2e-test resourceVersion: "20027" uid: ee252fc7-07b5-435e-9606-7edcea6ffae3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-37e3de-predictor-8557c96d5-d7527 namespace: kserve-ci-e2e-test resourceVersion: "19223" uid: 634a823b-50a1-443f-862b-e5afaa378932 kind: Event lastTimestamp: "2026-07-10T19:00:41Z" message: 'Readiness probe failed: Get "https://10.132.0.31:8643/healthz": dial tcp 10.132.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:00:41Z" name: isvc-primary-37e3de-predictor-8557c96d5-d7527.18c102d50ebffd0c namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: b034a7df-b6db-4143-939c-26d337c0e892 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:59:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-37e3de-predictor-8557c96d5 namespace: kserve-ci-e2e-test resourceVersion: "19197" uid: 6e123594-09b4-4538-b2dd-f5fc4067af74 kind: Event lastTimestamp: "2026-07-10T18:59:04Z" message: 'Created pod: isvc-primary-37e3de-predictor-8557c96d5-d7527' metadata: creationTimestamp: "2026-07-10T18:59:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:59:04Z" name: isvc-primary-37e3de-predictor-8557c96d5.18c102be637dc148 namespace: kserve-ci-e2e-test resourceVersion: "19224" uid: 2d130db5-c8d2-49b6-8505-668a7f1bee4b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:59:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-37e3de-predictor namespace: kserve-ci-e2e-test resourceVersion: "19194" uid: cd4e85e7-a535-48c0-80ee-9bd66f17643e kind: Event lastTimestamp: "2026-07-10T18:59:04Z" message: Scaled up replica set isvc-primary-37e3de-predictor-8557c96d5 from 0 to 1 metadata: creationTimestamp: "2026-07-10T18:59:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:59:04Z" name: isvc-primary-37e3de-predictor.18c102be5d566632 namespace: kserve-ci-e2e-test resourceVersion: "19199" uid: debac5c6-aee1-465e-9635-c41849f74a07 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T18:59:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-37e3de-predictor namespace: kserve-ci-e2e-test resourceVersion: "19198" uid: a522b81c-baf5-4ddc-95df-7f5a8608ac09 kind: Event lastTimestamp: "2026-07-10T18:59:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T18:59:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:59:49Z" name: isvc-primary-37e3de-predictor.18c102c1dc641bc1 namespace: kserve-ci-e2e-test resourceVersion: "19580" uid: 4cbaa5ed-fae0-4d07-bbde-0e421063f9ec reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T18:59:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-37e3de-predictor namespace: kserve-ci-e2e-test resourceVersion: "19198" uid: a522b81c-baf5-4ddc-95df-7f5a8608ac09 kind: Event lastTimestamp: "2026-07-10T18:59:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T18:59:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:59:49Z" name: isvc-primary-37e3de-predictor.18c102c1dcd384b5 namespace: kserve-ci-e2e-test resourceVersion: "19581" uid: ca135ee1-9424-4079-bb9d-7c9eeba3f878 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:00:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-37e3de-predictor namespace: kserve-ci-e2e-test resourceVersion: "19369" uid: a522b81c-baf5-4ddc-95df-7f5a8608ac09 kind: Event lastTimestamp: "2026-07-10T19:00:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:00:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:00:34Z" name: isvc-primary-37e3de-predictor.18c102cc5a50c405 namespace: kserve-ci-e2e-test resourceVersion: "19974" uid: 88e9b323-8b4f-44db-b7bb-22a38318cb1a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:00:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-37e3de-predictor namespace: kserve-ci-e2e-test resourceVersion: "19369" uid: a522b81c-baf5-4ddc-95df-7f5a8608ac09 kind: Event lastTimestamp: "2026-07-10T19:00:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:00:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:00:34Z" name: isvc-primary-37e3de-predictor.18c102cc5aa82811 namespace: kserve-ci-e2e-test resourceVersion: "19975" uid: 6b2649a7-c13a-4100-9dbc-3ead96d03f37 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-10T18:59:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-37e3de namespace: kserve-ci-e2e-test resourceVersion: "19186" uid: e65b4895-5c75-4be8-ab01-fb35c19c2d8f kind: Event lastTimestamp: "2026-07-10T19:00:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T18:59:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:00:16Z" name: isvc-primary-37e3de.18c102be5da0db49 namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: c267a35a-5cda-471d-931a-2c1a84802b84 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:59:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-37e3de namespace: kserve-ci-e2e-test resourceVersion: "19186" uid: e65b4895-5c75-4be8-ab01-fb35c19c2d8f kind: Event lastTimestamp: "2026-07-10T18:59:04Z" message: 'Failed to update status for InferenceService "isvc-primary-37e3de": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-37e3de": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T18:59:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:59:04Z" name: isvc-primary-37e3de.18c102be5e4c5ad7 namespace: kserve-ci-e2e-test resourceVersion: "19202" uid: b9bcc2be-887c-4948-ae9d-b26a7fed21c8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:59:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-37e3de namespace: kserve-ci-e2e-test resourceVersion: "19186" uid: e65b4895-5c75-4be8-ab01-fb35c19c2d8f kind: Event lastTimestamp: "2026-07-10T18:59:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-37e3de": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T18:59:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:59:04Z" name: isvc-primary-37e3de.18c102be5e4c9d3d namespace: kserve-ci-e2e-test resourceVersion: "19204" uid: fa4efeb2-3030-4085-808e-837b75b79ed4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-37e3de namespace: kserve-ci-e2e-test resourceVersion: "19775" uid: e65b4895-5c75-4be8-ab01-fb35c19c2d8f kind: Event lastTimestamp: "2026-07-10T19:00:16Z" message: InferenceService [isvc-primary-37e3de] is Ready metadata: creationTimestamp: "2026-07-10T19:00:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:00:16Z" name: isvc-primary-37e3de.18c102cf427dfa47 namespace: kserve-ci-e2e-test resourceVersion: "19776" uid: d30de006-1a89-4163-9429-16c06e807466 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T18:49:22.819880Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr namespace: kserve-ci-e2e-test resourceVersion: "14180" uid: 7650eb2c-498d-42d8-b5df-23614a22a871 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr to ip-10-0-134-57.ec2.internal metadata: creationTimestamp: "2026-07-10T18:49:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T18:49:22Z" name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr.18c10237096afa25 namespace: kserve-ci-e2e-test resourceVersion: "14186" uid: 67f99f50-153d-4bb6-be42-e451e45c651b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-856b4cf847-czcjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:49:23Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7650eb2c-498d-42d8-b5df-23614a22a871 kind: Event lastTimestamp: "2026-07-10T18:49:23Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-raw-sklearn-batcher-bc128-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T18:49:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:49:23Z" name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr.18c1023715e2cda2 namespace: kserve-ci-e2e-test resourceVersion: "14194" uid: bcec6194-b7b3-4ce2-a96d-47e129014099 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:49:23Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr namespace: kserve-ci-e2e-test resourceVersion: "14190" uid: 7650eb2c-498d-42d8-b5df-23614a22a871 kind: Event lastTimestamp: "2026-07-10T18:49:23Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T18:49:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T18:49:23Z" name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr.18c1023747597fbe namespace: kserve-ci-e2e-test resourceVersion: "14202" uid: 6f7d33bd-f1d9-4f02-9afd-99a7c5e7b76d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:49:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7650eb2c-498d-42d8-b5df-23614a22a871 kind: Event lastTimestamp: "2026-07-10T18:49:23Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:e7d18b8520466922c73925c5601004b31b3c3ec90f89a0821d13bdeee1848812" metadata: creationTimestamp: "2026-07-10T18:49:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:49:23Z" name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr.18c1023748dde0d2 namespace: kserve-ci-e2e-test resourceVersion: "14204" uid: 8934bf1b-51d8-4fe4-9c4f-913b8b0ef9bb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:49:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7650eb2c-498d-42d8-b5df-23614a22a871 kind: Event lastTimestamp: "2026-07-10T18:49:27Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:e7d18b8520466922c73925c5601004b31b3c3ec90f89a0821d13bdeee1848812" in 3.13s (3.13s including waiting). Image size: 302769600 bytes.' metadata: creationTimestamp: "2026-07-10T18:49:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:49:27Z" name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr.18c1023803757957 namespace: kserve-ci-e2e-test resourceVersion: "14229" uid: 40c765e0-2072-4b7f-ac77-67a2dcbf334a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:49:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7650eb2c-498d-42d8-b5df-23614a22a871 kind: Event lastTimestamp: "2026-07-10T18:49:27Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T18:49:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:49:27Z" name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr.18c1023806fc4926 namespace: kserve-ci-e2e-test resourceVersion: "14230" uid: a5029317-1f78-4337-999f-0923e83b17b4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:49:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7650eb2c-498d-42d8-b5df-23614a22a871 kind: Event lastTimestamp: "2026-07-10T18:49:27Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T18:49:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:49:27Z" name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr.18c1023807773d92 namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: ca0c5b32-2347-4230-af3c-09bd34e6c7a7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:49:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7650eb2c-498d-42d8-b5df-23614a22a871 kind: Event lastTimestamp: "2026-07-10T18:49:30Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1724" metadata: creationTimestamp: "2026-07-10T18:49:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:49:30Z" name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr.18c10238ecfa0f0f namespace: kserve-ci-e2e-test resourceVersion: "14268" uid: 7fe00c09-dc8f-4680-b488-c5d4be3e18d5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:49:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7650eb2c-498d-42d8-b5df-23614a22a871 kind: Event lastTimestamp: "2026-07-10T18:49:44Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1724" in 13.851s (13.851s including waiting). Image size: 1560931772 bytes.' metadata: creationTimestamp: "2026-07-10T18:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:49:44Z" name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr.18c1023c269d0bc2 namespace: kserve-ci-e2e-test resourceVersion: "14374" uid: bbf17e60-52ad-4a6b-8440-57e552810e6a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:49:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7650eb2c-498d-42d8-b5df-23614a22a871 kind: Event lastTimestamp: "2026-07-10T18:49:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T18:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:49:44Z" name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr.18c1023c2a1afb4f namespace: kserve-ci-e2e-test resourceVersion: "14375" uid: 048d8348-bcbb-4a1d-b02b-5a7bb9c82194 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:49:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7650eb2c-498d-42d8-b5df-23614a22a871 kind: Event lastTimestamp: "2026-07-10T18:49:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T18:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:49:44Z" name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr.18c1023c2a9cda54 namespace: kserve-ci-e2e-test resourceVersion: "14377" uid: 1d5fd74b-f12b-40e1-aa63-01fdb83dad94 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:49:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7650eb2c-498d-42d8-b5df-23614a22a871 kind: Event lastTimestamp: "2026-07-10T18:49:44Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-10T18:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:49:44Z" name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr.18c1023c2aad66bb namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: b719591d-7e9a-4f3f-94c2-4529490b8a22 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:49:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7650eb2c-498d-42d8-b5df-23614a22a871 kind: Event lastTimestamp: "2026-07-10T18:49:47Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.321s (2.321s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-10T18:49:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:49:47Z" name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr.18c1023cb50c505e namespace: kserve-ci-e2e-test resourceVersion: "14394" uid: a48fb042-f217-4b25-9400-5bd7de57b5df reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:49:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7650eb2c-498d-42d8-b5df-23614a22a871 kind: Event lastTimestamp: "2026-07-10T18:49:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T18:49:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:49:47Z" name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr.18c1023cb8f21f30 namespace: kserve-ci-e2e-test resourceVersion: "14395" uid: ce47572a-0185-4a14-9843-0fcee01a5c38 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:49:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7650eb2c-498d-42d8-b5df-23614a22a871 kind: Event lastTimestamp: "2026-07-10T18:49:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T18:49:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:49:47Z" name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr.18c1023cb976ab28 namespace: kserve-ci-e2e-test resourceVersion: "14396" uid: 63fae63f-aef4-4624-9c83-c2162f07af12 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:49:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7650eb2c-498d-42d8-b5df-23614a22a871 kind: Event lastTimestamp: "2026-07-10T18:49:47Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:f89d75505dfa0eae68aea417576d9edc02c9c812117fbdf23c2a565323101ab5" metadata: creationTimestamp: "2026-07-10T18:49:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:49:47Z" name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr.18c1023cb9859d65 namespace: kserve-ci-e2e-test resourceVersion: "14397" uid: 3494383c-1243-4356-aac6-f8f809fbb2bf reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:49:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7650eb2c-498d-42d8-b5df-23614a22a871 kind: Event lastTimestamp: "2026-07-10T18:49:49Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:f89d75505dfa0eae68aea417576d9edc02c9c812117fbdf23c2a565323101ab5" in 2.433s (2.433s including waiting). Image size: 239695460 bytes.' metadata: creationTimestamp: "2026-07-10T18:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:49:49Z" name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr.18c1023d4a931318 namespace: kserve-ci-e2e-test resourceVersion: "14415" uid: 158808c9-b27b-4e7c-8689-63cff7acdf09 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:49:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7650eb2c-498d-42d8-b5df-23614a22a871 kind: Event lastTimestamp: "2026-07-10T18:49:49Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-10T18:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:49:49Z" name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr.18c1023d4da28af3 namespace: kserve-ci-e2e-test resourceVersion: "14417" uid: 5a42c1b5-fdb7-47b4-b751-a5ddb4b9168f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:49:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7650eb2c-498d-42d8-b5df-23614a22a871 kind: Event lastTimestamp: "2026-07-10T18:49:49Z" message: Started container agent metadata: creationTimestamp: "2026-07-10T18:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:49:49Z" name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr.18c1023d4e351553 namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: 314a768e-f076-4859-b19d-368ddae546fd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-10T18:49:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7650eb2c-498d-42d8-b5df-23614a22a871 kind: Event lastTimestamp: "2026-07-10T18:51:26Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T18:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:26Z" name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr.18c1023d6007e035 namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: dbe2c749-c470-42ef-865f-9a9245cd3800 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-10T18:49:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7650eb2c-498d-42d8-b5df-23614a22a871 kind: Event lastTimestamp: "2026-07-10T18:51:26Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-10T18:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:26Z" name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr.18c1023d60135e23 namespace: kserve-ci-e2e-test resourceVersion: "15260" uid: a4225614-f248-4e22-b98b-0c1787883cbd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7650eb2c-498d-42d8-b5df-23614a22a871 kind: Event lastTimestamp: "2026-07-10T18:51:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T18:51:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:07Z" name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr.18c1024f7de06ed2 namespace: kserve-ci-e2e-test resourceVersion: "15041" uid: e55c7477-d2b2-4c32-9bf8-c33235ee3f35 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7650eb2c-498d-42d8-b5df-23614a22a871 kind: Event lastTimestamp: "2026-07-10T18:51:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T18:51:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:07Z" name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr.18c1024f7de7e1a7 namespace: kserve-ci-e2e-test resourceVersion: "15047" uid: 2c492777-3c14-427f-8153-73a3552fa084 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7650eb2c-498d-42d8-b5df-23614a22a871 kind: Event lastTimestamp: "2026-07-10T18:51:07Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-10T18:51:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:07Z" name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr.18c1024f7debc87d namespace: kserve-ci-e2e-test resourceVersion: "15052" uid: cc58e1f9-43ca-4c06-9d98-b96e613f822e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T18:51:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7650eb2c-498d-42d8-b5df-23614a22a871 kind: Event lastTimestamp: "2026-07-10T18:51:26Z" message: 'Readiness probe failed: Get "https://10.132.0.24:8643/healthz": dial tcp 10.132.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T18:51:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:26Z" name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr.18c102503c3484bf namespace: kserve-ci-e2e-test resourceVersion: "15257" uid: e1b7ea20-5c6e-46c9-bffc-82095d830c6f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:49:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8 namespace: kserve-ci-e2e-test resourceVersion: "14167" uid: 52c54c59-2d91-4f0f-ba41-b28d4618f81d kind: Event lastTimestamp: "2026-07-10T18:49:22Z" message: 'Created pod: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8-ktdvr' metadata: creationTimestamp: "2026-07-10T18:49:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:49:22Z" name: isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8.18c1023708fda5a1 namespace: kserve-ci-e2e-test resourceVersion: "14181" uid: 4c1b766f-4e83-4142-a3b0-c944ea0a0e68 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:49:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-bc128-predictor namespace: kserve-ci-e2e-test resourceVersion: "14166" uid: 6da40f78-dead-48b3-bfa3-f1693df57b3f kind: Event lastTimestamp: "2026-07-10T18:49:22Z" message: Scaled up replica set isvc-raw-sklearn-batcher-bc128-predictor-5c8d8d67d8 from 0 to 1 metadata: creationTimestamp: "2026-07-10T18:49:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:49:22Z" name: isvc-raw-sklearn-batcher-bc128-predictor.18c10237003f54c6 namespace: kserve-ci-e2e-test resourceVersion: "14168" uid: f63d4cbc-5341-4be0-8bf8-3714c8bfeb9c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T18:49:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-bc128-predictor namespace: kserve-ci-e2e-test resourceVersion: "14174" uid: 6075888f-188e-4ebe-9911-eb45d3ea1797 kind: Event lastTimestamp: "2026-07-10T18:50:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T18:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:50:22Z" name: isvc-raw-sklearn-batcher-bc128-predictor.18c1023a858de709 namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: 32d1ed20-6fbc-4c72-86f9-df8ae370ef11 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T18:49:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-bc128-predictor namespace: kserve-ci-e2e-test resourceVersion: "14174" uid: 6075888f-188e-4ebe-9911-eb45d3ea1797 kind: Event lastTimestamp: "2026-07-10T18:50:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T18:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:50:22Z" name: isvc-raw-sklearn-batcher-bc128-predictor.18c1023a85f20eaa namespace: kserve-ci-e2e-test resourceVersion: "14663" uid: 9c43de98-64d2-473c-b3e3-9ceed7c696a7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T18:50:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-bc128-predictor namespace: kserve-ci-e2e-test resourceVersion: "14323" uid: 6075888f-188e-4ebe-9911-eb45d3ea1797 kind: Event lastTimestamp: "2026-07-10T18:50:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T18:50:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:50:52Z" name: isvc-raw-sklearn-batcher-bc128-predictor.18c1024183fce407 namespace: kserve-ci-e2e-test resourceVersion: "14884" uid: 8260f3a9-d4d4-4d64-9355-d4133ba008b6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T18:50:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-bc128-predictor namespace: kserve-ci-e2e-test resourceVersion: "14323" uid: 6075888f-188e-4ebe-9911-eb45d3ea1797 kind: Event lastTimestamp: "2026-07-10T18:50:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T18:50:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:50:52Z" name: isvc-raw-sklearn-batcher-bc128-predictor.18c102418461ec5a namespace: kserve-ci-e2e-test resourceVersion: "14885" uid: e9bc37a9-ab26-4c78-b17c-460899e7cdca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-10T18:49:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-bc128 namespace: kserve-ci-e2e-test resourceVersion: "14158" uid: c82c45c1-f872-47ed-89e1-294dd7df98e2 kind: Event lastTimestamp: "2026-07-10T18:50:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T18:49:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:50:56Z" name: isvc-raw-sklearn-batcher-bc128.18c10237065861d2 namespace: kserve-ci-e2e-test resourceVersion: "14915" uid: dd9a3588-2f44-47e6-aa61-4257e699c091 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:49:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-bc128 namespace: kserve-ci-e2e-test resourceVersion: "14158" uid: c82c45c1-f872-47ed-89e1-294dd7df98e2 kind: Event lastTimestamp: "2026-07-10T18:49:22Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-bc128": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-bc128": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T18:49:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:49:22Z" name: isvc-raw-sklearn-batcher-bc128.18c1023706cad0ec namespace: kserve-ci-e2e-test resourceVersion: "14176" uid: ead4bf31-e9c3-44dc-8b7c-6ceb4554147a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:49:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-bc128 namespace: kserve-ci-e2e-test resourceVersion: "14158" uid: c82c45c1-f872-47ed-89e1-294dd7df98e2 kind: Event lastTimestamp: "2026-07-10T18:49:22Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-bc128": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T18:49:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:49:22Z" name: isvc-raw-sklearn-batcher-bc128.18c1023706cb22f4 namespace: kserve-ci-e2e-test resourceVersion: "14177" uid: 9b777b25-a9f4-407b-9e72-5c6be2b57fa0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:50:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-bc128 namespace: kserve-ci-e2e-test resourceVersion: "14912" uid: c82c45c1-f872-47ed-89e1-294dd7df98e2 kind: Event lastTimestamp: "2026-07-10T18:50:56Z" message: InferenceService [isvc-raw-sklearn-batcher-bc128] is Ready metadata: creationTimestamp: "2026-07-10T18:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:50:56Z" name: isvc-raw-sklearn-batcher-bc128.18c1024cc4e200f8 namespace: kserve-ci-e2e-test resourceVersion: "14913" uid: cb1b66b8-860e-4079-bbd9-92a4083f10bc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:00:24.225949Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-37e3de-predictor-696b97fb74-lh44x namespace: kserve-ci-e2e-test resourceVersion: "19861" uid: 04800502-8c3f-4563-bd50-52f40971c65d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-37e3de-predictor-696b97fb74-lh44x to ip-10-0-134-57.ec2.internal metadata: creationTimestamp: "2026-07-10T19:00:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:00:24Z" name: isvc-secondary-37e3de-predictor-696b97fb74-lh44x.18c102d1084ab0d2 namespace: kserve-ci-e2e-test resourceVersion: "19867" uid: e28365a9-17f4-45a5-8209-33aadc1b0521 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-856b4cf847-czcjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:24Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-37e3de-predictor-696b97fb74-lh44x namespace: kserve-ci-e2e-test resourceVersion: "19863" uid: 04800502-8c3f-4563-bd50-52f40971c65d kind: Event lastTimestamp: "2026-07-10T19:00:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-secondary-37e3de-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:00:24Z" name: isvc-secondary-37e3de-predictor-696b97fb74-lh44x.18c102d115c719c4 namespace: kserve-ci-e2e-test resourceVersion: "19878" uid: 791ab238-8f4f-4df9-b59f-a7390e44be3f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:25Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-37e3de-predictor-696b97fb74-lh44x namespace: kserve-ci-e2e-test resourceVersion: "19872" uid: 04800502-8c3f-4563-bd50-52f40971c65d kind: Event lastTimestamp: "2026-07-10T19:00:25Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:00:25Z" name: isvc-secondary-37e3de-predictor-696b97fb74-lh44x.18c102d14620d019 namespace: kserve-ci-e2e-test resourceVersion: "19885" uid: 6406d5ce-0419-4dea-af43-51d1ac3363cc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:00:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-37e3de-predictor-696b97fb74-lh44x namespace: kserve-ci-e2e-test resourceVersion: "19863" uid: 04800502-8c3f-4563-bd50-52f40971c65d kind: Event lastTimestamp: "2026-07-10T19:00:27Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:e7d18b8520466922c73925c5601004b31b3c3ec90f89a0821d13bdeee1848812" already present on machine metadata: creationTimestamp: "2026-07-10T19:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:00:27Z" name: isvc-secondary-37e3de-predictor-696b97fb74-lh44x.18c102d14827c3d1 namespace: kserve-ci-e2e-test resourceVersion: "19912" uid: 7f26f1a5-0604-4e14-b69a-ba55701b7d18 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:00:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-37e3de-predictor-696b97fb74-lh44x namespace: kserve-ci-e2e-test resourceVersion: "19863" uid: 04800502-8c3f-4563-bd50-52f40971c65d kind: Event lastTimestamp: "2026-07-10T19:00:27Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:00:27Z" name: isvc-secondary-37e3de-predictor-696b97fb74-lh44x.18c102d14b639400 namespace: kserve-ci-e2e-test resourceVersion: "19922" uid: 1c2474bd-af28-4194-858b-753260aad8de reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:00:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-37e3de-predictor-696b97fb74-lh44x namespace: kserve-ci-e2e-test resourceVersion: "19863" uid: 04800502-8c3f-4563-bd50-52f40971c65d kind: Event lastTimestamp: "2026-07-10T19:00:27Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:00:27Z" name: isvc-secondary-37e3de-predictor-696b97fb74-lh44x.18c102d14be2368f namespace: kserve-ci-e2e-test resourceVersion: "19924" uid: 29607ab9-c3b2-4a84-a6d2-41a5a0a241a6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-37e3de-predictor-696b97fb74-lh44x namespace: kserve-ci-e2e-test resourceVersion: "19863" uid: 04800502-8c3f-4563-bd50-52f40971c65d kind: Event lastTimestamp: "2026-07-10T19:00:32Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-37e3de-predictor-696b97fb74-lh44x_kserve-ci-e2e-test(04800502-8c3f-4563-bd50-52f40971c65d) metadata: creationTimestamp: "2026-07-10T19:00:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:00:32Z" name: isvc-secondary-37e3de-predictor-696b97fb74-lh44x.18c102d30cad9484 namespace: kserve-ci-e2e-test resourceVersion: "19962" uid: b6b6499e-dd9f-4989-8c85-56633a7afcd7 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-37e3de-predictor-696b97fb74 namespace: kserve-ci-e2e-test resourceVersion: "19854" uid: a765d6ad-b5a1-4572-aba3-9cd99a249435 kind: Event lastTimestamp: "2026-07-10T19:00:24Z" message: 'Created pod: isvc-secondary-37e3de-predictor-696b97fb74-lh44x' metadata: creationTimestamp: "2026-07-10T19:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:00:24Z" name: isvc-secondary-37e3de-predictor-696b97fb74.18c102d107cbe81b namespace: kserve-ci-e2e-test resourceVersion: "19862" uid: 8435a91c-b453-490e-ab60-fc3aa6b5361f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-37e3de-predictor namespace: kserve-ci-e2e-test resourceVersion: "19852" uid: e7bbf073-3255-459e-8c83-767cb869adcf kind: Event lastTimestamp: "2026-07-10T19:00:24Z" message: Scaled up replica set isvc-secondary-37e3de-predictor-696b97fb74 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:00:24Z" name: isvc-secondary-37e3de-predictor.18c102d104d9ab41 namespace: kserve-ci-e2e-test resourceVersion: "19856" uid: 145018a4-2381-4ec8-ac7b-7a47b53f1c9c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-10T19:00:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-37e3de namespace: kserve-ci-e2e-test resourceVersion: "19844" uid: 5f9b7102-af12-4e17-ac2a-ef68a5e19553 kind: Event lastTimestamp: "2026-07-10T19:00:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:00:34Z" name: isvc-secondary-37e3de.18c102d104978551 namespace: kserve-ci-e2e-test resourceVersion: "19979" uid: 6b0c0230-bc06-4bd1-ba73-f637025cba9b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-37e3de namespace: kserve-ci-e2e-test resourceVersion: "19844" uid: 5f9b7102-af12-4e17-ac2a-ef68a5e19553 kind: Event lastTimestamp: "2026-07-10T19:00:24Z" message: 'Failed to update status for InferenceService "isvc-secondary-37e3de": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-37e3de": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:00:24Z" name: isvc-secondary-37e3de.18c102d10507fa5a namespace: kserve-ci-e2e-test resourceVersion: "19858" uid: 383a96a3-c912-4977-86e4-3141d311b9c2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-37e3de namespace: kserve-ci-e2e-test resourceVersion: "19844" uid: 5f9b7102-af12-4e17-ac2a-ef68a5e19553 kind: Event lastTimestamp: "2026-07-10T19:00:24Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-37e3de": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:00:24Z" name: isvc-secondary-37e3de.18c102d1050841f9 namespace: kserve-ci-e2e-test resourceVersion: "19859" uid: 560ac7e9-396d-48f5-9427-2e8190b22ec8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-10T18:51:07.851592Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q namespace: kserve-ci-e2e-test resourceVersion: "15031" uid: 87bb0cba-50b1-4c39-b042-aebe1bb0076b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q to ip-10-0-134-57.ec2.internal metadata: creationTimestamp: "2026-07-10T18:51:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T18:51:07Z" name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q.18c1024f7dcbab1f namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: f879dd3a-e585-483a-bf3a-6aa45ec939d7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-856b4cf847-czcjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:08Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q namespace: kserve-ci-e2e-test resourceVersion: "15057" uid: 87bb0cba-50b1-4c39-b042-aebe1bb0076b kind: Event lastTimestamp: "2026-07-10T18:51:08Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T18:51:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T18:51:08Z" name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q.18c1024f98cd0788 namespace: kserve-ci-e2e-test resourceVersion: "15082" uid: e950f7e8-853b-4b08-8881-28640ca7e11c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: 87bb0cba-50b1-4c39-b042-aebe1bb0076b kind: Event lastTimestamp: "2026-07-10T18:51:08Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:e7d18b8520466922c73925c5601004b31b3c3ec90f89a0821d13bdeee1848812" already present on machine metadata: creationTimestamp: "2026-07-10T18:51:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:08Z" name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q.18c1024f9a519ed2 namespace: kserve-ci-e2e-test resourceVersion: "15085" uid: 888adf44-4087-45f4-9691-deca3296bc34 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: 87bb0cba-50b1-4c39-b042-aebe1bb0076b kind: Event lastTimestamp: "2026-07-10T18:51:08Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T18:51:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:08Z" name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q.18c1024f9cfc13bd namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: 2fa2596a-39ff-4c62-81d7-49f0284cefb9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: 87bb0cba-50b1-4c39-b042-aebe1bb0076b kind: Event lastTimestamp: "2026-07-10T18:51:08Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T18:51:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:08Z" name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q.18c1024f9d6f9f32 namespace: kserve-ci-e2e-test resourceVersion: "15089" uid: 8a942667-55b3-4e4a-a94d-4d294dcb080c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: 87bb0cba-50b1-4c39-b042-aebe1bb0076b kind: Event lastTimestamp: "2026-07-10T18:51:13Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1724" already present on machine metadata: creationTimestamp: "2026-07-10T18:51:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:13Z" name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q.18c10250c96f5739 namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: d7bf1863-4252-4e1d-a216-7b90d85cc5fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: 87bb0cba-50b1-4c39-b042-aebe1bb0076b kind: Event lastTimestamp: "2026-07-10T18:51:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T18:51:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:13Z" name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q.18c10250cc3a1231 namespace: kserve-ci-e2e-test resourceVersion: "15144" uid: 3bf69a4b-2191-4837-9685-90e2bcc7e996 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: 87bb0cba-50b1-4c39-b042-aebe1bb0076b kind: Event lastTimestamp: "2026-07-10T18:51:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T18:51:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:13Z" name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q.18c10250cca7573f namespace: kserve-ci-e2e-test resourceVersion: "15145" uid: f61795a1-0d4e-4c9e-a408-1277f66aeb38 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: 87bb0cba-50b1-4c39-b042-aebe1bb0076b kind: Event lastTimestamp: "2026-07-10T18:51:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T18:51:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:13Z" name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q.18c10250ccb38478 namespace: kserve-ci-e2e-test resourceVersion: "15146" uid: e1e3d7d2-ed05-4d6e-ac81-6e8fe8bf904d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: 87bb0cba-50b1-4c39-b042-aebe1bb0076b kind: Event lastTimestamp: "2026-07-10T18:51:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T18:51:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:13Z" name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q.18c10250cfefd845 namespace: kserve-ci-e2e-test resourceVersion: "15150" uid: 071805b5-b797-4782-aa4e-f6e87770f1dd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: 87bb0cba-50b1-4c39-b042-aebe1bb0076b kind: Event lastTimestamp: "2026-07-10T18:51:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T18:51:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:13Z" name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q.18c10250d061272e namespace: kserve-ci-e2e-test resourceVersion: "15152" uid: 494c68a9-576c-4aeb-94c0-3ae7a1f37328 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-10T18:51:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: 87bb0cba-50b1-4c39-b042-aebe1bb0076b kind: Event lastTimestamp: "2026-07-10T18:52:51Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T18:51:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:51Z" name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q.18c1025141642e4a namespace: kserve-ci-e2e-test resourceVersion: "16110" uid: 56aa7fca-79b4-4d63-b4d9-9f3e5f22ca18 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: 87bb0cba-50b1-4c39-b042-aebe1bb0076b kind: Event lastTimestamp: "2026-07-10T18:52:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T18:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:48Z" name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q.18c10266d9d3a572 namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: 94d31126-aa15-41d4-80dc-84d0cf64777e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: 87bb0cba-50b1-4c39-b042-aebe1bb0076b kind: Event lastTimestamp: "2026-07-10T18:52:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T18:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:48Z" name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q.18c10266d9d4f959 namespace: kserve-ci-e2e-test resourceVersion: "16007" uid: e1f1f55c-62fa-4ab9-a3bf-30fc46bdc0fb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: 87bb0cba-50b1-4c39-b042-aebe1bb0076b kind: Event lastTimestamp: "2026-07-10T18:52:51Z" message: 'Readiness probe failed: Get "https://10.132.0.25:8643/healthz": dial tcp 10.132.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T18:52:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:51Z" name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q.18c102679ba54db3 namespace: kserve-ci-e2e-test resourceVersion: "16109" uid: e901f76e-42a4-4280-97dc-1ff58ec8266c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd namespace: kserve-ci-e2e-test resourceVersion: "15004" uid: abc73b35-840e-479b-b7a6-bb445d22c85d kind: Event lastTimestamp: "2026-07-10T18:51:07Z" message: 'Created pod: isvc-sklearn-graph-raw-c446e-predictor-7579489dd-wwq8q' metadata: creationTimestamp: "2026-07-10T18:51:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:51:07Z" name: isvc-sklearn-graph-raw-c446e-predictor-7579489dd.18c1024f7d436625 namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 9680417c-fe0e-4af9-80a0-7c871d5391ff reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-c446e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15002" uid: 58ec1bf9-31fe-4b2c-8780-13ae737023c1 kind: Event lastTimestamp: "2026-07-10T18:51:07Z" message: Scaled up replica set isvc-sklearn-graph-raw-c446e-predictor-7579489dd from 0 to 1 metadata: creationTimestamp: "2026-07-10T18:51:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:51:07Z" name: isvc-sklearn-graph-raw-c446e-predictor.18c1024f78be3d6d namespace: kserve-ci-e2e-test resourceVersion: "15006" uid: b39130d5-2d90-49e4-9934-ca3e7ea8d147 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T18:51:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-c446e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: b38648d6-53fa-49b3-a40f-8cfaafa05952 kind: Event lastTimestamp: "2026-07-10T18:51:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T18:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:51:37Z" name: isvc-sklearn-graph-raw-c446e-predictor.18c10252f7c71df3 namespace: kserve-ci-e2e-test resourceVersion: "15347" uid: 6eb79302-17fe-4ce1-8ec4-353454061553 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T18:51:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-c446e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: b38648d6-53fa-49b3-a40f-8cfaafa05952 kind: Event lastTimestamp: "2026-07-10T18:51:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T18:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:51:37Z" name: isvc-sklearn-graph-raw-c446e-predictor.18c10252f828876a namespace: kserve-ci-e2e-test resourceVersion: "15348" uid: 12f3e880-6e22-40f0-a4ea-52ca4715d96c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T18:51:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-c446e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15237" uid: b38648d6-53fa-49b3-a40f-8cfaafa05952 kind: Event lastTimestamp: "2026-07-10T18:52:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T18:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:52:37Z" name: isvc-sklearn-graph-raw-c446e-predictor.18c10259f65b409f namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: 724fd718-625d-46e2-96fc-26b62d74c6fc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T18:51:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-c446e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15237" uid: b38648d6-53fa-49b3-a40f-8cfaafa05952 kind: Event lastTimestamp: "2026-07-10T18:52:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T18:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:52:37Z" name: isvc-sklearn-graph-raw-c446e-predictor.18c10259f6bfef69 namespace: kserve-ci-e2e-test resourceVersion: "15842" uid: b424285a-e887-4d30-a41e-ad4564a6c43d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-10T18:51:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-c446e namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: dbdecef6-c285-442b-866c-df1a7d2d3e6b kind: Event lastTimestamp: "2026-07-10T18:52:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T18:51:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:52:21Z" name: isvc-sklearn-graph-raw-c446e.18c1024f7890dd4b namespace: kserve-ci-e2e-test resourceVersion: "15696" uid: e3514115-b054-4042-9f6f-060a6cf0cd1c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-c446e namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: dbdecef6-c285-442b-866c-df1a7d2d3e6b kind: Event lastTimestamp: "2026-07-10T18:51:07Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-c446e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-c446e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T18:51:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:51:07Z" name: isvc-sklearn-graph-raw-c446e.18c1024f78f004c7 namespace: kserve-ci-e2e-test resourceVersion: "15008" uid: cdc0a08d-6436-4f22-9a74-80f42c679ef4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-c446e namespace: kserve-ci-e2e-test resourceVersion: "15694" uid: dbdecef6-c285-442b-866c-df1a7d2d3e6b kind: Event lastTimestamp: "2026-07-10T18:52:21Z" message: InferenceService [isvc-sklearn-graph-raw-c446e] is Ready metadata: creationTimestamp: "2026-07-10T18:52:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:52:21Z" name: isvc-sklearn-graph-raw-c446e.18c10260a521aba7 namespace: kserve-ci-e2e-test resourceVersion: "15695" uid: 1913e548-cc72-45f5-b650-5e181d44a4c0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T18:52:48.167224Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: 17def5e7-a73d-4836-a10d-a4b8dd37a860 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb to ip-10-0-134-57.ec2.internal metadata: creationTimestamp: "2026-07-10T18:52:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T18:52:48Z" name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb.18c10266d912d0dd namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 1e9abd87-5a1a-4c6e-b8b9-10a7285aa74c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-856b4cf847-czcjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:48Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb namespace: kserve-ci-e2e-test resourceVersion: "16011" uid: 17def5e7-a73d-4836-a10d-a4b8dd37a860 kind: Event lastTimestamp: "2026-07-10T18:52:48Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T18:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T18:52:48Z" name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb.18c10266f382b826 namespace: kserve-ci-e2e-test resourceVersion: "16055" uid: 3d722b00-bd43-4132-9bbf-cb46931b312b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: 17def5e7-a73d-4836-a10d-a4b8dd37a860 kind: Event lastTimestamp: "2026-07-10T18:52:48Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:e7d18b8520466922c73925c5601004b31b3c3ec90f89a0821d13bdeee1848812" already present on machine metadata: creationTimestamp: "2026-07-10T18:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:48Z" name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb.18c10266f564973f namespace: kserve-ci-e2e-test resourceVersion: "16057" uid: d98dc9c7-3038-42ee-a7ed-7b72040ca2f6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: 17def5e7-a73d-4836-a10d-a4b8dd37a860 kind: Event lastTimestamp: "2026-07-10T18:52:48Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T18:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:48Z" name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb.18c10266f7f6d83c namespace: kserve-ci-e2e-test resourceVersion: "16060" uid: 573e1bdb-cfc1-4ce1-85df-412e070cc788 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: 17def5e7-a73d-4836-a10d-a4b8dd37a860 kind: Event lastTimestamp: "2026-07-10T18:52:48Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T18:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:48Z" name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb.18c10266f86387f2 namespace: kserve-ci-e2e-test resourceVersion: "16061" uid: fec910b1-4c15-474f-bafe-650ca114c067 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: 17def5e7-a73d-4836-a10d-a4b8dd37a860 kind: Event lastTimestamp: "2026-07-10T18:52:52Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1724" already present on machine metadata: creationTimestamp: "2026-07-10T18:52:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:52Z" name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb.18c10267f0a83ed3 namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: b8b6e37b-bf02-4227-857a-7b270c66aa7f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: 17def5e7-a73d-4836-a10d-a4b8dd37a860 kind: Event lastTimestamp: "2026-07-10T18:52:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T18:52:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:52Z" name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb.18c10267f4d4acfd namespace: kserve-ci-e2e-test resourceVersion: "16139" uid: 46563c59-0e64-43dd-8190-2214da219008 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: 17def5e7-a73d-4836-a10d-a4b8dd37a860 kind: Event lastTimestamp: "2026-07-10T18:52:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T18:52:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:52Z" name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb.18c10267f54d036b namespace: kserve-ci-e2e-test resourceVersion: "16140" uid: 9e983f50-7a3d-407e-8759-333d97bc6fcc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: 17def5e7-a73d-4836-a10d-a4b8dd37a860 kind: Event lastTimestamp: "2026-07-10T18:52:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T18:52:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:52Z" name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb.18c10267f5595caa namespace: kserve-ci-e2e-test resourceVersion: "16141" uid: 23721c8a-2615-41bc-b869-a9fbb1a61ad2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: 17def5e7-a73d-4836-a10d-a4b8dd37a860 kind: Event lastTimestamp: "2026-07-10T18:52:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T18:52:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:52Z" name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb.18c10267f84986c5 namespace: kserve-ci-e2e-test resourceVersion: "16143" uid: 3d79d95a-d506-4761-afd3-5e1379678d28 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: 17def5e7-a73d-4836-a10d-a4b8dd37a860 kind: Event lastTimestamp: "2026-07-10T18:52:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T18:52:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:52Z" name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb.18c10267f8bf4c93 namespace: kserve-ci-e2e-test resourceVersion: "16144" uid: 476710e9-3937-4e83-a07f-6e79dc8f66f2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-10T18:52:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: 17def5e7-a73d-4836-a10d-a4b8dd37a860 kind: Event lastTimestamp: "2026-07-10T18:54:39Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T18:52:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:54:39Z" name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb.18c102682cdd0e96 namespace: kserve-ci-e2e-test resourceVersion: "17093" uid: ba37bb29-ac27-4b5c-86b2-10d2a4d432f7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: 17def5e7-a73d-4836-a10d-a4b8dd37a860 kind: Event lastTimestamp: "2026-07-10T18:54:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T18:54:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:54:38Z" name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb.18c102808c2b2a21 namespace: kserve-ci-e2e-test resourceVersion: "17050" uid: 3112e83d-4ba1-4f80-8e56-7c861f59b27a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: 17def5e7-a73d-4836-a10d-a4b8dd37a860 kind: Event lastTimestamp: "2026-07-10T18:54:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T18:54:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:54:38Z" name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb.18c102808c2bfe10 namespace: kserve-ci-e2e-test resourceVersion: "17053" uid: 94edbfed-b6c6-4518-9247-51ebe4fd0267 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: 17def5e7-a73d-4836-a10d-a4b8dd37a860 kind: Event lastTimestamp: "2026-07-10T18:54:39Z" message: 'Readiness probe failed: Get "https://10.132.0.27:8643/healthz": dial tcp 10.132.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T18:54:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:54:39Z" name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb.18c10280db3642e6 namespace: kserve-ci-e2e-test resourceVersion: "17091" uid: 0e630cbb-65c0-4635-993c-9c042e4ca466 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8 namespace: kserve-ci-e2e-test resourceVersion: "15963" uid: a969664f-db68-4020-b7cb-743fc6b6e097 kind: Event lastTimestamp: "2026-07-10T18:52:48Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8-jx4fb' metadata: creationTimestamp: "2026-07-10T18:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:52:48Z" name: isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8.18c10266d89847d5 namespace: kserve-ci-e2e-test resourceVersion: "15986" uid: 0ad4025b-efa4-410e-afab-bf2552db3286 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-e9877-predictor namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: e2dad746-44a4-44e2-b5fa-d99a06108ce0 kind: Event lastTimestamp: "2026-07-10T18:52:48Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-e9877-predictor-689f848dd8 from 0 to 1 metadata: creationTimestamp: "2026-07-10T18:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:52:48Z" name: isvc-sklearn-graph-raw-hpa-e9877-predictor.18c10266d4208124 namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 54ca57d4-8fb2-4515-9840-f9e35cd79930 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T18:53:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-e9877-predictor namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: ca064306-847d-4e53-b011-7c0d4f029c79 kind: Event lastTimestamp: "2026-07-10T18:53:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T18:53:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:53:33Z" name: isvc-sklearn-graph-raw-hpa-e9877-predictor.18c1026a531034f1 namespace: kserve-ci-e2e-test resourceVersion: "16457" uid: ec8a7e27-dc16-469e-ab83-08fb7aa1d64e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T18:53:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-e9877-predictor namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: ca064306-847d-4e53-b011-7c0d4f029c79 kind: Event lastTimestamp: "2026-07-10T18:53:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T18:53:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:53:33Z" name: isvc-sklearn-graph-raw-hpa-e9877-predictor.18c1026a53734c88 namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: 6c86aef3-bee7-4b02-95df-8a755420c483 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T18:53:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-e9877-predictor namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: ca064306-847d-4e53-b011-7c0d4f029c79 kind: Event lastTimestamp: "2026-07-10T18:54:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T18:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:54:33Z" name: isvc-sklearn-graph-raw-hpa-e9877-predictor.18c10274d0473dc3 namespace: kserve-ci-e2e-test resourceVersion: "16938" uid: 0c30b477-d945-48ee-9807-bc81c66742af reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T18:53:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-e9877-predictor namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: ca064306-847d-4e53-b011-7c0d4f029c79 kind: Event lastTimestamp: "2026-07-10T18:54:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T18:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:54:33Z" name: isvc-sklearn-graph-raw-hpa-e9877-predictor.18c10274d0b2b000 namespace: kserve-ci-e2e-test resourceVersion: "16939" uid: 433ae6c9-5d17-459c-a38d-a2c61b13a065 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-10T18:52:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-e9877 namespace: kserve-ci-e2e-test resourceVersion: "15952" uid: 8e7bcce4-d403-4ed1-af36-ac2069aa7f47 kind: Event lastTimestamp: "2026-07-10T18:54:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T18:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:54:09Z" name: isvc-sklearn-graph-raw-hpa-e9877.18c10266d3e9048d namespace: kserve-ci-e2e-test resourceVersion: "16741" uid: 13aec3aa-01ab-402b-acd1-fa7c7e30b362 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-e9877 namespace: kserve-ci-e2e-test resourceVersion: "15952" uid: 8e7bcce4-d403-4ed1-af36-ac2069aa7f47 kind: Event lastTimestamp: "2026-07-10T18:52:48Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-e9877": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-e9877": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T18:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:52:48Z" name: isvc-sklearn-graph-raw-hpa-e9877.18c10266d450ad36 namespace: kserve-ci-e2e-test resourceVersion: "15967" uid: 1cf5d9ac-ed00-48a6-9cd3-978e37bb7f90 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-e9877 namespace: kserve-ci-e2e-test resourceVersion: "16739" uid: 8e7bcce4-d403-4ed1-af36-ac2069aa7f47 kind: Event lastTimestamp: "2026-07-10T18:54:09Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-e9877] is Ready metadata: creationTimestamp: "2026-07-10T18:54:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:54:09Z" name: isvc-sklearn-graph-raw-hpa-e9877.18c10279e42cb847 namespace: kserve-ci-e2e-test resourceVersion: "16740" uid: 95cfb4e3-2fe9-42b0-b3a0-b5f109d2f19a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T18:56:23.788402Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5 namespace: kserve-ci-e2e-test resourceVersion: "17982" uid: 1dd825eb-0697-4d5f-a131-5be830283941 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5 to ip-10-0-134-57.ec2.internal metadata: creationTimestamp: "2026-07-10T18:56:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T18:56:23Z" name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5.18c102990d18dd4e namespace: kserve-ci-e2e-test resourceVersion: "17988" uid: 8820f5b9-2995-4b3d-9d39-7c58c75cfd68 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-856b4cf847-czcjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:56:24Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5 namespace: kserve-ci-e2e-test resourceVersion: "17992" uid: 1dd825eb-0697-4d5f-a131-5be830283941 kind: Event lastTimestamp: "2026-07-10T18:56:24Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T18:56:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T18:56:24Z" name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5.18c102992719dd6d namespace: kserve-ci-e2e-test resourceVersion: "18018" uid: be82ce05-d4ca-4dd8-a06d-8526f2ec9bec reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:56:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5 namespace: kserve-ci-e2e-test resourceVersion: "17984" uid: 1dd825eb-0697-4d5f-a131-5be830283941 kind: Event lastTimestamp: "2026-07-10T18:56:24Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:e7d18b8520466922c73925c5601004b31b3c3ec90f89a0821d13bdeee1848812" already present on machine metadata: creationTimestamp: "2026-07-10T18:56:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:56:24Z" name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5.18c10299289859c2 namespace: kserve-ci-e2e-test resourceVersion: "18021" uid: 8d5111a7-89b3-4a5d-b442-e10054230195 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:56:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5 namespace: kserve-ci-e2e-test resourceVersion: "17984" uid: 1dd825eb-0697-4d5f-a131-5be830283941 kind: Event lastTimestamp: "2026-07-10T18:56:24Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T18:56:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:56:24Z" name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5.18c102992c1182fa namespace: kserve-ci-e2e-test resourceVersion: "18024" uid: 9dd35cbe-08cc-4185-9f08-6fd1c327b30d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:56:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5 namespace: kserve-ci-e2e-test resourceVersion: "17984" uid: 1dd825eb-0697-4d5f-a131-5be830283941 kind: Event lastTimestamp: "2026-07-10T18:56:24Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T18:56:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:56:24Z" name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5.18c102992c8508ff namespace: kserve-ci-e2e-test resourceVersion: "18025" uid: 647814c2-4feb-4e8c-8aec-de9f1693d001 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:56:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5 namespace: kserve-ci-e2e-test resourceVersion: "17984" uid: 1dd825eb-0697-4d5f-a131-5be830283941 kind: Event lastTimestamp: "2026-07-10T18:56:28Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1724" already present on machine metadata: creationTimestamp: "2026-07-10T18:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:56:28Z" name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5.18c1029a3889daa7 namespace: kserve-ci-e2e-test resourceVersion: "18070" uid: 69828d91-0ac6-41ac-840b-a1f4c5860cf3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:56:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5 namespace: kserve-ci-e2e-test resourceVersion: "17984" uid: 1dd825eb-0697-4d5f-a131-5be830283941 kind: Event lastTimestamp: "2026-07-10T18:56:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T18:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:56:28Z" name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5.18c1029a3baeb356 namespace: kserve-ci-e2e-test resourceVersion: "18072" uid: 4e594961-e307-4061-a5f9-6593df7d5b64 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:56:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5 namespace: kserve-ci-e2e-test resourceVersion: "17984" uid: 1dd825eb-0697-4d5f-a131-5be830283941 kind: Event lastTimestamp: "2026-07-10T18:56:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T18:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:56:28Z" name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5.18c1029a3c2727fb namespace: kserve-ci-e2e-test resourceVersion: "18073" uid: 24c171f4-cf1c-46a8-ae56-15a34aface19 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:56:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5 namespace: kserve-ci-e2e-test resourceVersion: "17984" uid: 1dd825eb-0697-4d5f-a131-5be830283941 kind: Event lastTimestamp: "2026-07-10T18:56:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T18:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:56:28Z" name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5.18c1029a3c32890d namespace: kserve-ci-e2e-test resourceVersion: "18075" uid: a448fae2-d4fd-4388-9967-ac53d865d29b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:56:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5 namespace: kserve-ci-e2e-test resourceVersion: "17984" uid: 1dd825eb-0697-4d5f-a131-5be830283941 kind: Event lastTimestamp: "2026-07-10T18:56:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T18:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:56:28Z" name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5.18c1029a3f9c403d namespace: kserve-ci-e2e-test resourceVersion: "18077" uid: 6c741521-d31e-49ac-94c6-67f70bf5ed7c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:56:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5 namespace: kserve-ci-e2e-test resourceVersion: "17984" uid: 1dd825eb-0697-4d5f-a131-5be830283941 kind: Event lastTimestamp: "2026-07-10T18:56:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T18:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:56:28Z" name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5.18c1029a4017951e namespace: kserve-ci-e2e-test resourceVersion: "18079" uid: 84c24a5e-e76d-478c-a980-58dd9e342f8f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 17 eventTime: null firstTimestamp: "2026-07-10T18:56:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5 namespace: kserve-ci-e2e-test resourceVersion: "17984" uid: 1dd825eb-0697-4d5f-a131-5be830283941 kind: Event lastTimestamp: "2026-07-10T18:59:05Z" message: 'Readiness probe failed: dial tcp 10.132.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T18:56:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:59:05Z" name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5.18c1029a74940ad7 namespace: kserve-ci-e2e-test resourceVersion: "19257" uid: e063ac16-9ad5-4cab-8e9c-19466ff9b63f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:59:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5 namespace: kserve-ci-e2e-test resourceVersion: "17984" uid: 1dd825eb-0697-4d5f-a131-5be830283941 kind: Event lastTimestamp: "2026-07-10T18:59:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T18:59:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:59:04Z" name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5.18c102be64b19f91 namespace: kserve-ci-e2e-test resourceVersion: "19230" uid: b1582676-6f4a-497e-8152-18d4d3d6bd32 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:59:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5 namespace: kserve-ci-e2e-test resourceVersion: "17984" uid: 1dd825eb-0697-4d5f-a131-5be830283941 kind: Event lastTimestamp: "2026-07-10T18:59:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T18:59:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:59:04Z" name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5.18c102be64b2d3f1 namespace: kserve-ci-e2e-test resourceVersion: "19231" uid: 65aa4d3e-4de8-4120-8a90-d4c30d27194e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T18:59:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5 namespace: kserve-ci-e2e-test resourceVersion: "17984" uid: 1dd825eb-0697-4d5f-a131-5be830283941 kind: Event lastTimestamp: "2026-07-10T18:59:10Z" message: 'Readiness probe failed: Get "https://10.132.0.30:8643/healthz": dial tcp 10.132.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T18:59:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:59:10Z" name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5.18c102bec70e1a61 namespace: kserve-ci-e2e-test resourceVersion: "19313" uid: 41865d52-c654-4eed-bfba-1dc4ca9eb593 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:56:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478 namespace: kserve-ci-e2e-test resourceVersion: "17964" uid: 14670c85-a619-497d-b5c2-85a413998c7e kind: Event lastTimestamp: "2026-07-10T18:56:23Z" message: 'Created pod: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478-qj8m5' metadata: creationTimestamp: "2026-07-10T18:56:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:56:23Z" name: isvc-sklearn-scale-raw-f5a60-predictor-596cc79478.18c102990c938ab9 namespace: kserve-ci-e2e-test resourceVersion: "17985" uid: 6c190584-8cc6-4eca-b4bd-88587af083e7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:56:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-f5a60-predictor namespace: kserve-ci-e2e-test resourceVersion: "17956" uid: 5dda7d1d-dc3d-4bbb-86ca-f48655c1b705 kind: Event lastTimestamp: "2026-07-10T18:56:23Z" message: Scaled up replica set isvc-sklearn-scale-raw-f5a60-predictor-596cc79478 from 0 to 1 metadata: creationTimestamp: "2026-07-10T18:56:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:56:23Z" name: isvc-sklearn-scale-raw-f5a60-predictor.18c1029908977936 namespace: kserve-ci-e2e-test resourceVersion: "17969" uid: eafb4457-f867-4dd5-8406-49c551d96dce reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T18:56:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-f5a60-predictor namespace: kserve-ci-e2e-test resourceVersion: "17963" uid: 782cf62c-bb6e-478b-8ed3-6bbc257bfaa4 kind: Event lastTimestamp: "2026-07-10T18:56:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T18:56:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:56:53Z" name: isvc-sklearn-scale-raw-f5a60-predictor.18c1029c876d5eab namespace: kserve-ci-e2e-test resourceVersion: "18266" uid: 53e805a9-bbf7-4040-a312-58bd16774c16 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T18:56:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-f5a60-predictor namespace: kserve-ci-e2e-test resourceVersion: "17963" uid: 782cf62c-bb6e-478b-8ed3-6bbc257bfaa4 kind: Event lastTimestamp: "2026-07-10T18:56:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T18:56:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:56:53Z" name: isvc-sklearn-scale-raw-f5a60-predictor.18c1029c87fd2342 namespace: kserve-ci-e2e-test resourceVersion: "18267" uid: 0641ce4b-3ebd-48e0-9ece-546a4cff5ba4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-10T18:57:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-f5a60-predictor namespace: kserve-ci-e2e-test resourceVersion: "18153" uid: 782cf62c-bb6e-478b-8ed3-6bbc257bfaa4 kind: Event lastTimestamp: "2026-07-10T18:58:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T18:57:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:58:53Z" name: isvc-sklearn-scale-raw-f5a60-predictor.18c102a38637fcbf namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: f20db1db-3a51-417a-829a-ddba8ae0ed0b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-10T18:57:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-f5a60-predictor namespace: kserve-ci-e2e-test resourceVersion: "18153" uid: 782cf62c-bb6e-478b-8ed3-6bbc257bfaa4 kind: Event lastTimestamp: "2026-07-10T18:58:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T18:57:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:58:53Z" name: isvc-sklearn-scale-raw-f5a60-predictor.18c102a386bcf72a namespace: kserve-ci-e2e-test resourceVersion: "19103" uid: bccfb813-491e-44ea-9317-090c465a4fea reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-10T18:56:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-f5a60 namespace: kserve-ci-e2e-test resourceVersion: "17945" uid: 0dbb831d-7422-4e00-b4e2-eacedeeaef18 kind: Event lastTimestamp: "2026-07-10T18:58:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T18:56:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:58:55Z" name: isvc-sklearn-scale-raw-f5a60.18c10299087fedec namespace: kserve-ci-e2e-test resourceVersion: "19123" uid: 7b000dc2-7c6d-40f7-b297-e8b1674f492f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:56:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-f5a60 namespace: kserve-ci-e2e-test resourceVersion: "17945" uid: 0dbb831d-7422-4e00-b4e2-eacedeeaef18 kind: Event lastTimestamp: "2026-07-10T18:56:23Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-f5a60": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-f5a60": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T18:56:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:56:23Z" name: isvc-sklearn-scale-raw-f5a60.18c102990931aed1 namespace: kserve-ci-e2e-test resourceVersion: "17973" uid: e6b89d0a-f238-4479-aead-21e7f51ddcf0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:58:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-f5a60 namespace: kserve-ci-e2e-test resourceVersion: "19121" uid: 0dbb831d-7422-4e00-b4e2-eacedeeaef18 kind: Event lastTimestamp: "2026-07-10T18:58:55Z" message: InferenceService [isvc-sklearn-scale-raw-f5a60] is Ready metadata: creationTimestamp: "2026-07-10T18:58:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:58:55Z" name: isvc-sklearn-scale-raw-f5a60.18c102bc67ca4273 namespace: kserve-ci-e2e-test resourceVersion: "19122" uid: 26b0eaff-5f70-4f8c-8ed5-80444946ea82 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T18:51:07.930424Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb namespace: kserve-ci-e2e-test resourceVersion: "15060" uid: a9d16cb0-6067-4bb6-8ddd-6efe59db5464 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb to ip-10-0-129-211.ec2.internal metadata: creationTimestamp: "2026-07-10T18:51:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T18:51:07Z" name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb.18c1024f827e94e9 namespace: kserve-ci-e2e-test resourceVersion: "15066" uid: 74a8d597-9418-43e8-b7c2-1da0131b36a5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-856b4cf847-czcjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:08Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb namespace: kserve-ci-e2e-test resourceVersion: "15073" uid: a9d16cb0-6067-4bb6-8ddd-6efe59db5464 kind: Event lastTimestamp: "2026-07-10T18:51:08Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T18:51:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T18:51:08Z" name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb.18c1024f9c2bdc1e namespace: kserve-ci-e2e-test resourceVersion: "15086" uid: 08e3daeb-92d7-459b-9bf8-8964ad456dd0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb namespace: kserve-ci-e2e-test resourceVersion: "15063" uid: a9d16cb0-6067-4bb6-8ddd-6efe59db5464 kind: Event lastTimestamp: "2026-07-10T18:51:08Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:e7d18b8520466922c73925c5601004b31b3c3ec90f89a0821d13bdeee1848812" metadata: creationTimestamp: "2026-07-10T18:51:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:08Z" name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb.18c1024f9d94918d namespace: kserve-ci-e2e-test resourceVersion: "15090" uid: b9d6aa05-12e5-4332-90f6-ec2701431b96 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb namespace: kserve-ci-e2e-test resourceVersion: "15063" uid: a9d16cb0-6067-4bb6-8ddd-6efe59db5464 kind: Event lastTimestamp: "2026-07-10T18:51:11Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:e7d18b8520466922c73925c5601004b31b3c3ec90f89a0821d13bdeee1848812" in 3.508s (3.508s including waiting). Image size: 302769600 bytes.' metadata: creationTimestamp: "2026-07-10T18:51:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:11Z" name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb.18c102506eb3c99f namespace: kserve-ci-e2e-test resourceVersion: "15122" uid: 3712d9ed-fea2-414b-b589-b409c1abd308 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb namespace: kserve-ci-e2e-test resourceVersion: "15063" uid: a9d16cb0-6067-4bb6-8ddd-6efe59db5464 kind: Event lastTimestamp: "2026-07-10T18:51:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T18:51:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:11Z" name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb.18c1025072400470 namespace: kserve-ci-e2e-test resourceVersion: "15123" uid: 1a0ceb65-d6ef-4e59-aa3f-a3aaa0526935 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb namespace: kserve-ci-e2e-test resourceVersion: "15063" uid: a9d16cb0-6067-4bb6-8ddd-6efe59db5464 kind: Event lastTimestamp: "2026-07-10T18:51:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T18:51:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:11Z" name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb.18c1025072ac1249 namespace: kserve-ci-e2e-test resourceVersion: "15124" uid: 395049e9-9274-49ea-8d5c-ac9db778ae88 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb namespace: kserve-ci-e2e-test resourceVersion: "15063" uid: a9d16cb0-6067-4bb6-8ddd-6efe59db5464 kind: Event lastTimestamp: "2026-07-10T18:51:16Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-10T18:51:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:16Z" name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb.18c1025165f6e66d namespace: kserve-ci-e2e-test resourceVersion: "15172" uid: c43220f4-c20a-42db-8ed5-3109b949280b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb namespace: kserve-ci-e2e-test resourceVersion: "15063" uid: a9d16cb0-6067-4bb6-8ddd-6efe59db5464 kind: Event lastTimestamp: "2026-07-10T18:51:35Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 19.939s (19.939s including waiting). Image size: 1305567991 bytes.' metadata: creationTimestamp: "2026-07-10T18:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:35Z" name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb.18c102560a752369 namespace: kserve-ci-e2e-test resourceVersion: "15327" uid: 7ec526ce-1d65-488f-b299-619bcaf81504 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb namespace: kserve-ci-e2e-test resourceVersion: "15063" uid: a9d16cb0-6067-4bb6-8ddd-6efe59db5464 kind: Event lastTimestamp: "2026-07-10T18:51:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T18:51:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:36Z" name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb.18c102560dad18f2 namespace: kserve-ci-e2e-test resourceVersion: "15329" uid: 4bd9b48a-b891-46d8-aba5-f173b42cea13 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb namespace: kserve-ci-e2e-test resourceVersion: "15063" uid: a9d16cb0-6067-4bb6-8ddd-6efe59db5464 kind: Event lastTimestamp: "2026-07-10T18:51:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T18:51:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:36Z" name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb.18c102560e1de058 namespace: kserve-ci-e2e-test resourceVersion: "15330" uid: e4aa2f71-13dc-4fa5-8c5c-12c92cde86b9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb namespace: kserve-ci-e2e-test resourceVersion: "15063" uid: a9d16cb0-6067-4bb6-8ddd-6efe59db5464 kind: Event lastTimestamp: "2026-07-10T18:51:36Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-10T18:51:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:36Z" name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb.18c102560e2aaf66 namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: eb2ff1e6-fb0c-457e-b544-84a17f60afbc reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb namespace: kserve-ci-e2e-test resourceVersion: "15063" uid: a9d16cb0-6067-4bb6-8ddd-6efe59db5464 kind: Event lastTimestamp: "2026-07-10T18:51:38Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.162s (2.162s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-10T18:51:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:38Z" name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb.18c102568f179aa0 namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: a0c0644d-a49c-4f22-8c93-2d1d6723ab76 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb namespace: kserve-ci-e2e-test resourceVersion: "15063" uid: a9d16cb0-6067-4bb6-8ddd-6efe59db5464 kind: Event lastTimestamp: "2026-07-10T18:51:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T18:51:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:38Z" name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb.18c10256928944e6 namespace: kserve-ci-e2e-test resourceVersion: "15356" uid: ac5e0588-ce76-4f9c-9586-9a5604913282 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb namespace: kserve-ci-e2e-test resourceVersion: "15063" uid: a9d16cb0-6067-4bb6-8ddd-6efe59db5464 kind: Event lastTimestamp: "2026-07-10T18:51:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T18:51:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:51:38Z" name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb.18c1025693013406 namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: dc265482-0d19-46f6-a7cb-b46d2e4732dd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-10T18:51:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb namespace: kserve-ci-e2e-test resourceVersion: "15063" uid: a9d16cb0-6067-4bb6-8ddd-6efe59db5464 kind: Event lastTimestamp: "2026-07-10T18:52:25Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T18:51:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:25Z" name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb.18c10256c75b06b1 namespace: kserve-ci-e2e-test resourceVersion: "15724" uid: 7b78ba6b-5276-4b4d-a346-b01803d9417c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb namespace: kserve-ci-e2e-test resourceVersion: "15063" uid: a9d16cb0-6067-4bb6-8ddd-6efe59db5464 kind: Event lastTimestamp: "2026-07-10T18:52:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T18:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:48Z" name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb.18c10266e1ad49be namespace: kserve-ci-e2e-test resourceVersion: "16037" uid: ffddcb32-55bf-4d5e-8e15-9d9d778320c4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb namespace: kserve-ci-e2e-test resourceVersion: "15063" uid: a9d16cb0-6067-4bb6-8ddd-6efe59db5464 kind: Event lastTimestamp: "2026-07-10T18:52:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T18:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:48Z" name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb.18c10266e1ae8396 namespace: kserve-ci-e2e-test resourceVersion: "16041" uid: 198dd7ec-8d68-43e8-a800-303a2b4f560c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb namespace: kserve-ci-e2e-test resourceVersion: "15063" uid: a9d16cb0-6067-4bb6-8ddd-6efe59db5464 kind: Event lastTimestamp: "2026-07-10T18:52:50Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T18:52:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:50Z" name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb.18c102674f7bc639 namespace: kserve-ci-e2e-test resourceVersion: "16089" uid: 919bfe8a-0e84-4aa4-a3a1-42b8c3840d3d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5 namespace: kserve-ci-e2e-test resourceVersion: "15044" uid: c1d859fb-852c-4c31-b331-5761591ee31c kind: Event lastTimestamp: "2026-07-10T18:51:07Z" message: 'Created pod: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5-6m7fb' metadata: creationTimestamp: "2026-07-10T18:51:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:51:07Z" name: isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5.18c1024f81eb8b30 namespace: kserve-ci-e2e-test resourceVersion: "15062" uid: d3b69ae1-6c10-422c-b751-4dfdfe5f716c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-c446e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: 0f518c08-d789-4db2-914c-ad402a336315 kind: Event lastTimestamp: "2026-07-10T18:51:07Z" message: Scaled up replica set isvc-xgboost-graph-raw-c446e-predictor-547f5b47f5 from 0 to 1 metadata: creationTimestamp: "2026-07-10T18:51:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:51:07Z" name: isvc-xgboost-graph-raw-c446e-predictor.18c1024f7ef2aa14 namespace: kserve-ci-e2e-test resourceVersion: "15048" uid: d03acd56-4402-4f16-827d-a33f39ddd6e2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T18:51:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-c446e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15043" uid: a60976ee-d927-4c52-85ad-03daf7baf667 kind: Event lastTimestamp: "2026-07-10T18:52:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T18:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:52:07Z" name: isvc-xgboost-graph-raw-c446e-predictor.18c10252fd35d09e namespace: kserve-ci-e2e-test resourceVersion: "15595" uid: 69a58748-88a4-4d34-a79d-edba06fa3942 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T18:51:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-c446e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15043" uid: a60976ee-d927-4c52-85ad-03daf7baf667 kind: Event lastTimestamp: "2026-07-10T18:52:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T18:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:52:07Z" name: isvc-xgboost-graph-raw-c446e-predictor.18c10252fd9a53ed namespace: kserve-ci-e2e-test resourceVersion: "15596" uid: 3aa774d3-a641-4925-91b2-c8d38f1294aa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T18:52:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-c446e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15240" uid: a60976ee-d927-4c52-85ad-03daf7baf667 kind: Event lastTimestamp: "2026-07-10T18:52:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T18:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:52:37Z" name: isvc-xgboost-graph-raw-c446e-predictor.18c10260f863e7b4 namespace: kserve-ci-e2e-test resourceVersion: "15848" uid: be84b3af-cd68-4766-9094-67e903c88ca4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T18:52:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-c446e-predictor namespace: kserve-ci-e2e-test resourceVersion: "15240" uid: a60976ee-d927-4c52-85ad-03daf7baf667 kind: Event lastTimestamp: "2026-07-10T18:52:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T18:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:52:37Z" name: isvc-xgboost-graph-raw-c446e-predictor.18c10260f8c35b1f namespace: kserve-ci-e2e-test resourceVersion: "15850" uid: 1e0ccd4c-119d-40d5-801e-1ffd896fdef4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-10T18:51:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-c446e namespace: kserve-ci-e2e-test resourceVersion: "15010" uid: d1e124a6-f0fa-453a-916d-a94e01e82ff4 kind: Event lastTimestamp: "2026-07-10T18:52:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T18:51:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:52:35Z" name: isvc-xgboost-graph-raw-c446e.18c1024f7e851071 namespace: kserve-ci-e2e-test resourceVersion: "15799" uid: 539869a2-8315-496d-b929-bb196da3b965 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:51:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-c446e namespace: kserve-ci-e2e-test resourceVersion: "15010" uid: d1e124a6-f0fa-453a-916d-a94e01e82ff4 kind: Event lastTimestamp: "2026-07-10T18:51:07Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-c446e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-c446e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T18:51:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:51:07Z" name: isvc-xgboost-graph-raw-c446e.18c1024f7f50b0be namespace: kserve-ci-e2e-test resourceVersion: "15053" uid: a1e76865-8225-4786-909c-1b5a6d6574d6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-c446e namespace: kserve-ci-e2e-test resourceVersion: "15797" uid: d1e124a6-f0fa-453a-916d-a94e01e82ff4 kind: Event lastTimestamp: "2026-07-10T18:52:35Z" message: InferenceService [isvc-xgboost-graph-raw-c446e] is Ready metadata: creationTimestamp: "2026-07-10T18:52:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:52:35Z" name: isvc-xgboost-graph-raw-c446e.18c10263d6518202 namespace: kserve-ci-e2e-test resourceVersion: "15798" uid: 1309af43-7600-407f-9ebc-c4e640682835 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T18:52:48.312540Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps namespace: kserve-ci-e2e-test resourceVersion: "16030" uid: 50650f77-fac1-420b-a9fb-79e927ede74a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps to ip-10-0-129-211.ec2.internal metadata: creationTimestamp: "2026-07-10T18:52:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T18:52:48Z" name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps.18c10266e1bc1632 namespace: kserve-ci-e2e-test resourceVersion: "16038" uid: 200da3a9-eb85-4a22-9ab2-4d910f373e16 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-856b4cf847-czcjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:48Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps namespace: kserve-ci-e2e-test resourceVersion: "16048" uid: 50650f77-fac1-420b-a9fb-79e927ede74a kind: Event lastTimestamp: "2026-07-10T18:52:48Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T18:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T18:52:48Z" name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps.18c10266fbf3efad namespace: kserve-ci-e2e-test resourceVersion: "16066" uid: d2ac1d67-243b-41f2-b0e0-e3114a153959 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 50650f77-fac1-420b-a9fb-79e927ede74a kind: Event lastTimestamp: "2026-07-10T18:52:48Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:e7d18b8520466922c73925c5601004b31b3c3ec90f89a0821d13bdeee1848812" already present on machine metadata: creationTimestamp: "2026-07-10T18:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:48Z" name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps.18c10266fe40fefc namespace: kserve-ci-e2e-test resourceVersion: "16068" uid: 5a163d39-327e-4683-b21d-384ac0007c28 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 50650f77-fac1-420b-a9fb-79e927ede74a kind: Event lastTimestamp: "2026-07-10T18:52:48Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T18:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:48Z" name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps.18c102670123b633 namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 635ffade-2e92-4cd4-be7b-69b32c731d3d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 50650f77-fac1-420b-a9fb-79e927ede74a kind: Event lastTimestamp: "2026-07-10T18:52:48Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T18:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:48Z" name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps.18c102670192d40e namespace: kserve-ci-e2e-test resourceVersion: "16070" uid: 895bf386-cffd-464b-afad-aa33236b237b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 50650f77-fac1-420b-a9fb-79e927ede74a kind: Event lastTimestamp: "2026-07-10T18:52:52Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-07-10T18:52:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:52Z" name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps.18c10267d889bdf8 namespace: kserve-ci-e2e-test resourceVersion: "16120" uid: 9aa7e65b-9ba7-447b-ab1d-f4d87d68026b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 50650f77-fac1-420b-a9fb-79e927ede74a kind: Event lastTimestamp: "2026-07-10T18:52:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T18:52:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:52Z" name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps.18c10267dc5a656d namespace: kserve-ci-e2e-test resourceVersion: "16126" uid: d0317022-786b-4df9-89bc-97f576c52da3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 50650f77-fac1-420b-a9fb-79e927ede74a kind: Event lastTimestamp: "2026-07-10T18:52:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T18:52:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:52Z" name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps.18c10267dccbc2aa namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: bff95061-bf03-4788-a4ab-60ed8b5bf38c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 50650f77-fac1-420b-a9fb-79e927ede74a kind: Event lastTimestamp: "2026-07-10T18:52:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T18:52:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:52Z" name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps.18c10267dcd6a1b2 namespace: kserve-ci-e2e-test resourceVersion: "16128" uid: c230f1dc-decb-4efd-9f3b-2350d2318f4a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 50650f77-fac1-420b-a9fb-79e927ede74a kind: Event lastTimestamp: "2026-07-10T18:52:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T18:52:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:52Z" name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps.18c10267df82d7be namespace: kserve-ci-e2e-test resourceVersion: "16130" uid: 94073155-e778-4a2a-b110-1f8c56e0d78b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 50650f77-fac1-420b-a9fb-79e927ede74a kind: Event lastTimestamp: "2026-07-10T18:52:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T18:52:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:52Z" name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps.18c10267dff10a38 namespace: kserve-ci-e2e-test resourceVersion: "16131" uid: 60a2191f-6628-4945-9730-fec7c91fcfae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-10T18:52:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 50650f77-fac1-420b-a9fb-79e927ede74a kind: Event lastTimestamp: "2026-07-10T18:54:40Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T18:52:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:54:40Z" name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps.18c102685078301e namespace: kserve-ci-e2e-test resourceVersion: "17105" uid: 3c89934a-ed8c-4f41-b466-0e9916916141 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 50650f77-fac1-420b-a9fb-79e927ede74a kind: Event lastTimestamp: "2026-07-10T18:54:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T18:54:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:54:38Z" name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps.18c10280911e2965 namespace: kserve-ci-e2e-test resourceVersion: "17072" uid: 552e06bb-c3b8-42b5-b836-3fa3fe6b0795 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 50650f77-fac1-420b-a9fb-79e927ede74a kind: Event lastTimestamp: "2026-07-10T18:54:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T18:54:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:54:38Z" name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps.18c10280911e962b namespace: kserve-ci-e2e-test resourceVersion: "17073" uid: d19f6f3f-8130-406a-a613-35ce0035147a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 50650f77-fac1-420b-a9fb-79e927ede74a kind: Event lastTimestamp: "2026-07-10T18:54:40Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T18:54:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:54:40Z" name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps.18c10280febd7fc8 namespace: kserve-ci-e2e-test resourceVersion: "17104" uid: 3a162e57-1685-4c43-9929-d760d7810c0f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99 namespace: kserve-ci-e2e-test resourceVersion: "15998" uid: 1c095a4c-1d6d-406f-afed-771d4eb5d807 kind: Event lastTimestamp: "2026-07-10T18:52:48Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99-jl9ps' metadata: creationTimestamp: "2026-07-10T18:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:52:48Z" name: isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99.18c10266e10be80b namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: 57cdbcb9-5a87-48c7-928c-4d78a727f23d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-e9877-predictor namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: 124559d2-88f2-4196-a4db-77a84c817448 kind: Event lastTimestamp: "2026-07-10T18:52:48Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-e9877-predictor-6ccdd5f99 from 0 to 1 metadata: creationTimestamp: "2026-07-10T18:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:52:48Z" name: isvc-xgboost-graph-raw-hpa-e9877-predictor.18c10266da0c5d66 namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 91279a10-7c0b-458b-b75a-bedf244825c7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T18:53:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-e9877-predictor namespace: kserve-ci-e2e-test resourceVersion: "15996" uid: 1301731e-8919-40a5-9579-15a76c3c1f3a kind: Event lastTimestamp: "2026-07-10T18:53:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T18:53:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:53:33Z" name: isvc-xgboost-graph-raw-hpa-e9877-predictor.18c1026a587a6643 namespace: kserve-ci-e2e-test resourceVersion: "16460" uid: adf241a0-7140-4bea-8d64-3457cbcf1637 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T18:53:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-e9877-predictor namespace: kserve-ci-e2e-test resourceVersion: "15996" uid: 1301731e-8919-40a5-9579-15a76c3c1f3a kind: Event lastTimestamp: "2026-07-10T18:53:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T18:53:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:53:33Z" name: isvc-xgboost-graph-raw-hpa-e9877-predictor.18c1026a58dfa104 namespace: kserve-ci-e2e-test resourceVersion: "16461" uid: eefd0c76-b757-4b18-889e-9f8cd290bb37 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T18:53:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-e9877-predictor namespace: kserve-ci-e2e-test resourceVersion: "16239" uid: 1301731e-8919-40a5-9579-15a76c3c1f3a kind: Event lastTimestamp: "2026-07-10T18:54:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T18:53:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:54:18Z" name: isvc-xgboost-graph-raw-hpa-e9877-predictor.18c1026dd79c0968 namespace: kserve-ci-e2e-test resourceVersion: "16802" uid: 84d128c0-92cf-46f6-876c-3388f135e35d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T18:53:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-e9877-predictor namespace: kserve-ci-e2e-test resourceVersion: "16239" uid: 1301731e-8919-40a5-9579-15a76c3c1f3a kind: Event lastTimestamp: "2026-07-10T18:54:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T18:53:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:54:18Z" name: isvc-xgboost-graph-raw-hpa-e9877-predictor.18c1026dd801ac1b namespace: kserve-ci-e2e-test resourceVersion: "16803" uid: c3247d0f-1d74-47ec-bacb-4b091c40bd80 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-10T18:52:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-e9877 namespace: kserve-ci-e2e-test resourceVersion: "15969" uid: f7a4767b-b05a-4b93-9ff7-e8b447849312 kind: Event lastTimestamp: "2026-07-10T18:54:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T18:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:54:00Z" name: isvc-xgboost-graph-raw-hpa-e9877.18c10266d9dec0b9 namespace: kserve-ci-e2e-test resourceVersion: "16666" uid: e54d7254-ece3-4357-bce2-191f4b5dd5e6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-e9877 namespace: kserve-ci-e2e-test resourceVersion: "15969" uid: f7a4767b-b05a-4b93-9ff7-e8b447849312 kind: Event lastTimestamp: "2026-07-10T18:52:48Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-hpa-e9877": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-hpa-e9877": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T18:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:52:48Z" name: isvc-xgboost-graph-raw-hpa-e9877.18c10266da6fdaa5 namespace: kserve-ci-e2e-test resourceVersion: "16008" uid: e5d5dae4-3ab3-49ca-aa3a-ec65a49e5e88 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-e9877 namespace: kserve-ci-e2e-test resourceVersion: "16663" uid: f7a4767b-b05a-4b93-9ff7-e8b447849312 kind: Event lastTimestamp: "2026-07-10T18:54:00Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-e9877] is Ready metadata: creationTimestamp: "2026-07-10T18:54:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:54:00Z" name: isvc-xgboost-graph-raw-hpa-e9877.18c10277b3b10a2b namespace: kserve-ci-e2e-test resourceVersion: "16664" uid: 21594956-ec64-4ed3-875f-34881e084a75 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T18:54:38.525295Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-bb4f7-predictor-79d4f46b7c-cn262 namespace: kserve-ci-e2e-test resourceVersion: "17033" uid: 06b77dc0-9801-444d-846b-3cf91a61ce99 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-bb4f7-predictor-79d4f46b7c-cn262 to ip-10-0-129-211.ec2.internal metadata: creationTimestamp: "2026-07-10T18:54:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T18:54:38Z" name: message-dumper-raw-bb4f7-predictor-79d4f46b7c-cn262.18c102808aeda820 namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: 0ca44a80-aee8-4279-b910-8864a1926d60 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-856b4cf847-czcjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:38Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-bb4f7-predictor-79d4f46b7c-cn262 namespace: kserve-ci-e2e-test resourceVersion: "17052" uid: 06b77dc0-9801-444d-846b-3cf91a61ce99 kind: Event lastTimestamp: "2026-07-10T18:54:38Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T18:54:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T18:54:38Z" name: message-dumper-raw-bb4f7-predictor-79d4f46b7c-cn262.18c10280a4cf2d3c namespace: kserve-ci-e2e-test resourceVersion: "17081" uid: 6b48aacf-8db7-43ea-9544-3251689357d8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-bb4f7-predictor-79d4f46b7c-cn262 namespace: kserve-ci-e2e-test resourceVersion: "17038" uid: 06b77dc0-9801-444d-846b-3cf91a61ce99 kind: Event lastTimestamp: "2026-07-10T18:54:38Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-10T18:54:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:54:38Z" name: message-dumper-raw-bb4f7-predictor-79d4f46b7c-cn262.18c10280a651deea namespace: kserve-ci-e2e-test resourceVersion: "17083" uid: e8b2657e-b571-4b20-b7ee-47b0d3643f5d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-bb4f7-predictor-79d4f46b7c-cn262 namespace: kserve-ci-e2e-test resourceVersion: "17038" uid: 06b77dc0-9801-444d-846b-3cf91a61ce99 kind: Event lastTimestamp: "2026-07-10T18:54:40Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.26s (1.26s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-10T18:54:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:54:40Z" name: message-dumper-raw-bb4f7-predictor-79d4f46b7c-cn262.18c10280f1784b4b namespace: kserve-ci-e2e-test resourceVersion: "17095" uid: f6904547-976e-4c37-a3f0-085e4766f005 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-bb4f7-predictor-79d4f46b7c-cn262 namespace: kserve-ci-e2e-test resourceVersion: "17038" uid: 06b77dc0-9801-444d-846b-3cf91a61ce99 kind: Event lastTimestamp: "2026-07-10T18:54:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T18:54:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:54:40Z" name: message-dumper-raw-bb4f7-predictor-79d4f46b7c-cn262.18c10280f548861e namespace: kserve-ci-e2e-test resourceVersion: "17098" uid: 60edc754-9ab5-4454-a822-88c7f24ef1a0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-bb4f7-predictor-79d4f46b7c-cn262 namespace: kserve-ci-e2e-test resourceVersion: "17038" uid: 06b77dc0-9801-444d-846b-3cf91a61ce99 kind: Event lastTimestamp: "2026-07-10T18:54:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T18:54:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:54:40Z" name: message-dumper-raw-bb4f7-predictor-79d4f46b7c-cn262.18c10280f5bb275d namespace: kserve-ci-e2e-test resourceVersion: "17099" uid: 9ecb3ca4-d101-4839-aa50-cf728689c1db reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-bb4f7-predictor-79d4f46b7c-cn262 namespace: kserve-ci-e2e-test resourceVersion: "17038" uid: 06b77dc0-9801-444d-846b-3cf91a61ce99 kind: Event lastTimestamp: "2026-07-10T18:54:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T18:54:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:54:40Z" name: message-dumper-raw-bb4f7-predictor-79d4f46b7c-cn262.18c10280f5c534b2 namespace: kserve-ci-e2e-test resourceVersion: "17100" uid: 4b1f55bb-2b0f-4c17-a564-2f721c431d37 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-bb4f7-predictor-79d4f46b7c-cn262 namespace: kserve-ci-e2e-test resourceVersion: "17038" uid: 06b77dc0-9801-444d-846b-3cf91a61ce99 kind: Event lastTimestamp: "2026-07-10T18:54:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T18:54:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:54:40Z" name: message-dumper-raw-bb4f7-predictor-79d4f46b7c-cn262.18c10280f8f6ea99 namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 52093c9e-c412-44f8-8f71-f6866e102f45 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-bb4f7-predictor-79d4f46b7c-cn262 namespace: kserve-ci-e2e-test resourceVersion: "17038" uid: 06b77dc0-9801-444d-846b-3cf91a61ce99 kind: Event lastTimestamp: "2026-07-10T18:54:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T18:54:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:54:40Z" name: message-dumper-raw-bb4f7-predictor-79d4f46b7c-cn262.18c10280f9657371 namespace: kserve-ci-e2e-test resourceVersion: "17102" uid: f1d15eb7-26ae-42df-9c96-b8cf53a2622b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:56:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-bb4f7-predictor-79d4f46b7c-cn262 namespace: kserve-ci-e2e-test resourceVersion: "17038" uid: 06b77dc0-9801-444d-846b-3cf91a61ce99 kind: Event lastTimestamp: "2026-07-10T18:56:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T18:56:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:56:23Z" name: message-dumper-raw-bb4f7-predictor-79d4f46b7c-cn262.18c1029911e82fe3 namespace: kserve-ci-e2e-test resourceVersion: "18009" uid: b15bece3-2349-4a25-9acb-938365ea97e3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:56:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-bb4f7-predictor-79d4f46b7c-cn262 namespace: kserve-ci-e2e-test resourceVersion: "17038" uid: 06b77dc0-9801-444d-846b-3cf91a61ce99 kind: Event lastTimestamp: "2026-07-10T18:56:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T18:56:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:56:23Z" name: message-dumper-raw-bb4f7-predictor-79d4f46b7c-cn262.18c1029911e8c014 namespace: kserve-ci-e2e-test resourceVersion: "18010" uid: 083bf922-58bc-4901-8611-8dbf1d141bfe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:56:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-bb4f7-predictor-79d4f46b7c-cn262 namespace: kserve-ci-e2e-test resourceVersion: "17038" uid: 06b77dc0-9801-444d-846b-3cf91a61ce99 kind: Event lastTimestamp: "2026-07-10T18:56:23Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T18:56:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:56:23Z" name: message-dumper-raw-bb4f7-predictor-79d4f46b7c-cn262.18c10299158589f2 namespace: kserve-ci-e2e-test resourceVersion: "18014" uid: 51dcaab0-5783-4c5c-9e9d-bf327dfd15e9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-211.ec2.internal source: component: kubelet host: ip-10-0-129-211.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-bb4f7-predictor-79d4f46b7c namespace: kserve-ci-e2e-test resourceVersion: "17020" uid: d43e3961-2d6c-44ac-8a90-70d38245f042 kind: Event lastTimestamp: "2026-07-10T18:54:38Z" message: 'Created pod: message-dumper-raw-bb4f7-predictor-79d4f46b7c-cn262' metadata: creationTimestamp: "2026-07-10T18:54:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:54:38Z" name: message-dumper-raw-bb4f7-predictor-79d4f46b7c.18c102808a7166ee namespace: kserve-ci-e2e-test resourceVersion: "17039" uid: 9dfa169b-fa63-4bb4-becf-7343858eb5f0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-bb4f7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17018" uid: 3bd1c291-0a8a-4964-8a52-e9efcf96398b kind: Event lastTimestamp: "2026-07-10T18:54:38Z" message: Scaled up replica set message-dumper-raw-bb4f7-predictor-79d4f46b7c from 0 to 1 metadata: creationTimestamp: "2026-07-10T18:54:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:54:38Z" name: message-dumper-raw-bb4f7-predictor.18c10280866f9ba7 namespace: kserve-ci-e2e-test resourceVersion: "17022" uid: 80658f20-9642-4102-b6c8-4190e01308bc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-bb4f7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17019" uid: 6e403d46-0b7f-4dd6-9caa-f6616d442690 kind: Event lastTimestamp: "2026-07-10T18:54:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T18:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:54:53Z" name: message-dumper-raw-bb4f7-predictor.18c10284054234fa namespace: kserve-ci-e2e-test resourceVersion: "17215" uid: 7bbd3f2b-8416-47ea-81b0-184d5ce9ae91 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-bb4f7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17019" uid: 6e403d46-0b7f-4dd6-9caa-f6616d442690 kind: Event lastTimestamp: "2026-07-10T18:54:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T18:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:54:53Z" name: message-dumper-raw-bb4f7-predictor.18c1028405a9ee4c namespace: kserve-ci-e2e-test resourceVersion: "17217" uid: 5de99ac3-e521-4573-ad60-dc31cb883841 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:55:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-bb4f7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17216" uid: 6e403d46-0b7f-4dd6-9caa-f6616d442690 kind: Event lastTimestamp: "2026-07-10T18:55:08Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T18:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:55:08Z" name: message-dumper-raw-bb4f7-predictor.18c10287847aa863 namespace: kserve-ci-e2e-test resourceVersion: "17386" uid: 35f89d8d-058b-4dc5-829f-5fbd40237926 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:55:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-bb4f7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17216" uid: 6e403d46-0b7f-4dd6-9caa-f6616d442690 kind: Event lastTimestamp: "2026-07-10T18:55:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T18:55:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:55:08Z" name: message-dumper-raw-bb4f7-predictor.18c1028784fa4e5b namespace: kserve-ci-e2e-test resourceVersion: "17388" uid: faf508f2-5ad9-46a7-adc0-30703f9e47b7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-10T18:54:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-bb4f7 namespace: kserve-ci-e2e-test resourceVersion: "17008" uid: fe9a0d74-9ddc-4c8e-9d26-f74e6ecd93be kind: Event lastTimestamp: "2026-07-10T18:54:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T18:54:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:54:49Z" name: message-dumper-raw-bb4f7.18c10280864e8932 namespace: kserve-ci-e2e-test resourceVersion: "17188" uid: c3edbb4d-c91c-4f5c-b102-0b4b1bcf68c3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-bb4f7 namespace: kserve-ci-e2e-test resourceVersion: "17008" uid: fe9a0d74-9ddc-4c8e-9d26-f74e6ecd93be kind: Event lastTimestamp: "2026-07-10T18:54:38Z" message: 'Failed to update status for InferenceService "message-dumper-raw-bb4f7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-bb4f7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T18:54:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:54:38Z" name: message-dumper-raw-bb4f7.18c1028086c124d1 namespace: kserve-ci-e2e-test resourceVersion: "17024" uid: 47164551-db62-43c3-9599-b5c521aff786 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-bb4f7 namespace: kserve-ci-e2e-test resourceVersion: "17184" uid: fe9a0d74-9ddc-4c8e-9d26-f74e6ecd93be kind: Event lastTimestamp: "2026-07-10T18:54:49Z" message: InferenceService [message-dumper-raw-bb4f7] is Ready metadata: creationTimestamp: "2026-07-10T18:54:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:54:49Z" name: message-dumper-raw-bb4f7.18c10282fefceef0 namespace: kserve-ci-e2e-test resourceVersion: "17185" uid: 18e30b64-4027-45ff-90d3-5869241c3434 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T18:52:37.856056Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-c446e-55b9c8fd58-nh2z8 namespace: kserve-ci-e2e-test resourceVersion: "15829" uid: 3e32ae0e-27e9-4e83-9754-9e368651ab4a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-c446e-55b9c8fd58-nh2z8 to ip-10-0-134-57.ec2.internal metadata: creationTimestamp: "2026-07-10T18:52:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T18:52:37Z" name: model-chainer-raw-c446e-55b9c8fd58-nh2z8.18c10264727acc68 namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 012d5df4-1224-480e-a1b6-26c333579564 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-856b4cf847-czcjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:38Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-c446e-55b9c8fd58-nh2z8 namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 3e32ae0e-27e9-4e83-9754-9e368651ab4a kind: Event lastTimestamp: "2026-07-10T18:52:38Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T18:52:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T18:52:38Z" name: model-chainer-raw-c446e-55b9c8fd58-nh2z8.18c102648ca96527 namespace: kserve-ci-e2e-test resourceVersion: "15857" uid: 1c828df9-50dd-4a9b-9571-144c128af997 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c446e} kind: Pod name: model-chainer-raw-c446e-55b9c8fd58-nh2z8 namespace: kserve-ci-e2e-test resourceVersion: "15831" uid: 3e32ae0e-27e9-4e83-9754-9e368651ab4a kind: Event lastTimestamp: "2026-07-10T18:52:38Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:8b2b640cecca2c5abab08a86fa1fe077f2518209438c00eb88348417ca75cab6" metadata: creationTimestamp: "2026-07-10T18:52:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:38Z" name: model-chainer-raw-c446e-55b9c8fd58-nh2z8.18c102648de481cc namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: cd585e62-31d4-4fe9-a16d-0d854b1201f3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c446e} kind: Pod name: model-chainer-raw-c446e-55b9c8fd58-nh2z8 namespace: kserve-ci-e2e-test resourceVersion: "15831" uid: 3e32ae0e-27e9-4e83-9754-9e368651ab4a kind: Event lastTimestamp: "2026-07-10T18:52:40Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:8b2b640cecca2c5abab08a86fa1fe077f2518209438c00eb88348417ca75cab6" in 2.365s (2.365s including waiting). Image size: 218009189 bytes.' metadata: creationTimestamp: "2026-07-10T18:52:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:40Z" name: model-chainer-raw-c446e-55b9c8fd58-nh2z8.18c102651ae8d18f namespace: kserve-ci-e2e-test resourceVersion: "15876" uid: ffff753a-6e68-48d0-9afe-4a2f8dd4086f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c446e} kind: Pod name: model-chainer-raw-c446e-55b9c8fd58-nh2z8 namespace: kserve-ci-e2e-test resourceVersion: "15831" uid: 3e32ae0e-27e9-4e83-9754-9e368651ab4a kind: Event lastTimestamp: "2026-07-10T18:52:40Z" message: 'Created container: model-chainer-raw-c446e' metadata: creationTimestamp: "2026-07-10T18:52:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:40Z" name: model-chainer-raw-c446e-55b9c8fd58-nh2z8.18c102651e12ead3 namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: c352613d-7e68-4a9d-a927-1880e2bd7c51 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c446e} kind: Pod name: model-chainer-raw-c446e-55b9c8fd58-nh2z8 namespace: kserve-ci-e2e-test resourceVersion: "15831" uid: 3e32ae0e-27e9-4e83-9754-9e368651ab4a kind: Event lastTimestamp: "2026-07-10T18:52:40Z" message: Started container model-chainer-raw-c446e metadata: creationTimestamp: "2026-07-10T18:52:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:40Z" name: model-chainer-raw-c446e-55b9c8fd58-nh2z8.18c102651e7ae7ff namespace: kserve-ci-e2e-test resourceVersion: "15881" uid: abacc22e-12d5-4376-9e98-f746072428f6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c446e} kind: Pod name: model-chainer-raw-c446e-55b9c8fd58-nh2z8 namespace: kserve-ci-e2e-test resourceVersion: "15831" uid: 3e32ae0e-27e9-4e83-9754-9e368651ab4a kind: Event lastTimestamp: "2026-07-10T18:52:47Z" message: Stopping container model-chainer-raw-c446e metadata: creationTimestamp: "2026-07-10T18:52:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:52:47Z" name: model-chainer-raw-c446e-55b9c8fd58-nh2z8.18c10266cb3e72af namespace: kserve-ci-e2e-test resourceVersion: "15945" uid: 3f252fb2-8440-4e32-ab81-c1d181ae3851 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T18:52:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c446e} kind: Pod name: model-chainer-raw-c446e-55b9c8fd58-nh2z8 namespace: kserve-ci-e2e-test resourceVersion: "15831" uid: 3e32ae0e-27e9-4e83-9754-9e368651ab4a kind: Event lastTimestamp: "2026-07-10T18:53:16Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-10T18:52:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:53:16Z" name: model-chainer-raw-c446e-55b9c8fd58-nh2z8.18c10267b1e5699d namespace: kserve-ci-e2e-test resourceVersion: "16331" uid: 494ee7da-d2d3-49a4-a403-058984fdf6b8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-c446e-55b9c8fd58 namespace: kserve-ci-e2e-test resourceVersion: "15826" uid: e2ab7042-4e23-4b3d-b22b-2da78bfcca8d kind: Event lastTimestamp: "2026-07-10T18:52:37Z" message: 'Created pod: model-chainer-raw-c446e-55b9c8fd58-nh2z8' metadata: creationTimestamp: "2026-07-10T18:52:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:52:37Z" name: model-chainer-raw-c446e-55b9c8fd58.18c1026471f19c32 namespace: kserve-ci-e2e-test resourceVersion: "15830" uid: 7ee005f8-8595-433a-b61a-83f65e6a0cc4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-c446e namespace: kserve-ci-e2e-test resourceVersion: "15824" uid: 9a8cda7d-4822-41f3-b733-842b6282328a kind: Event lastTimestamp: "2026-07-10T18:52:37Z" message: Scaled up replica set model-chainer-raw-c446e-55b9c8fd58 from 0 to 1 metadata: creationTimestamp: "2026-07-10T18:52:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:52:37Z" name: model-chainer-raw-c446e.18c102647125fdef namespace: kserve-ci-e2e-test resourceVersion: "15828" uid: a7f794e7-1817-4f44-834f-2b33b64de0af reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-c446e namespace: kserve-ci-e2e-test resourceVersion: "15845" uid: 934f5a67-9ed9-4b3c-9a90-35f009a184d3 kind: Event lastTimestamp: "2026-07-10T18:52:37Z" message: InferenceGraph [model-chainer-raw-c446e] is Ready metadata: creationTimestamp: "2026-07-10T18:52:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:52:37Z" name: model-chainer-raw-c446e.18c10264762add46 namespace: kserve-ci-e2e-test resourceVersion: "15847" uid: 61d01b4a-1a0a-43bf-9280-e35a98e466b6 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-c446e namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: 934f5a67-9ed9-4b3c-9a90-35f009a184d3 kind: Event lastTimestamp: "2026-07-10T18:52:37Z" message: 'Failed to update status for InferenceGraph "model-chainer-raw-c446e": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer-raw-c446e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T18:52:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:52:37Z" name: model-chainer-raw-c446e.18c1026478621146 namespace: kserve-ci-e2e-test resourceVersion: "15851" uid: 582ce84f-f124-484a-b915-ef5699b21263 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:52:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-c446e namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: 934f5a67-9ed9-4b3c-9a90-35f009a184d3 kind: Event lastTimestamp: "2026-07-10T18:52:37Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer-raw-c446e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T18:52:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:52:37Z" name: model-chainer-raw-c446e.18c1026478626e26 namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: 54b38d53-a992-45fd-9eff-25be3b81ab28 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-10T18:54:28.240918Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-e9877-646885f67d-vwf6d namespace: kserve-ci-e2e-test resourceVersion: "16880" uid: 91eea447-c8b2-4279-8ec6-5c0583dba188 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-e9877-646885f67d-vwf6d to ip-10-0-134-57.ec2.internal metadata: creationTimestamp: "2026-07-10T18:54:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T18:54:28Z" name: model-chainer-raw-hpa-e9877-646885f67d-vwf6d.18c1027e25ee1a69 namespace: kserve-ci-e2e-test resourceVersion: "16885" uid: 817ca574-129a-4a9f-8a2f-f5c1b6bad7a5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-856b4cf847-czcjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:28Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-e9877-646885f67d-vwf6d namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: 91eea447-c8b2-4279-8ec6-5c0583dba188 kind: Event lastTimestamp: "2026-07-10T18:54:28Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T18:54:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T18:54:28Z" name: model-chainer-raw-hpa-e9877-646885f67d-vwf6d.18c1027e3fbcfe04 namespace: kserve-ci-e2e-test resourceVersion: "16900" uid: 448a0678-08fb-43e8-874a-3b753078ef0c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-e9877} kind: Pod name: model-chainer-raw-hpa-e9877-646885f67d-vwf6d namespace: kserve-ci-e2e-test resourceVersion: "16883" uid: 91eea447-c8b2-4279-8ec6-5c0583dba188 kind: Event lastTimestamp: "2026-07-10T18:54:28Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:8b2b640cecca2c5abab08a86fa1fe077f2518209438c00eb88348417ca75cab6" already present on machine metadata: creationTimestamp: "2026-07-10T18:54:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:54:28Z" name: model-chainer-raw-hpa-e9877-646885f67d-vwf6d.18c1027e423c3cf7 namespace: kserve-ci-e2e-test resourceVersion: "16903" uid: 635afabe-b0ba-4a1b-8f33-930792d9a30a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-e9877} kind: Pod name: model-chainer-raw-hpa-e9877-646885f67d-vwf6d namespace: kserve-ci-e2e-test resourceVersion: "16883" uid: 91eea447-c8b2-4279-8ec6-5c0583dba188 kind: Event lastTimestamp: "2026-07-10T18:54:28Z" message: 'Created container: model-chainer-raw-hpa-e9877' metadata: creationTimestamp: "2026-07-10T18:54:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:54:28Z" name: model-chainer-raw-hpa-e9877-646885f67d-vwf6d.18c1027e467ead51 namespace: kserve-ci-e2e-test resourceVersion: "16904" uid: 925ab824-a6d7-495f-a0b9-b26553f0ca36 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-e9877} kind: Pod name: model-chainer-raw-hpa-e9877-646885f67d-vwf6d namespace: kserve-ci-e2e-test resourceVersion: "16883" uid: 91eea447-c8b2-4279-8ec6-5c0583dba188 kind: Event lastTimestamp: "2026-07-10T18:54:28Z" message: Started container model-chainer-raw-hpa-e9877 metadata: creationTimestamp: "2026-07-10T18:54:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:54:28Z" name: model-chainer-raw-hpa-e9877-646885f67d-vwf6d.18c1027e46f914eb namespace: kserve-ci-e2e-test resourceVersion: "16905" uid: 66a2f0dc-0323-4bd5-800b-e5a548e9c7bb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-e9877} kind: Pod name: model-chainer-raw-hpa-e9877-646885f67d-vwf6d namespace: kserve-ci-e2e-test resourceVersion: "16883" uid: 91eea447-c8b2-4279-8ec6-5c0583dba188 kind: Event lastTimestamp: "2026-07-10T18:54:38Z" message: Stopping container model-chainer-raw-hpa-e9877 metadata: creationTimestamp: "2026-07-10T18:54:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:54:38Z" name: model-chainer-raw-hpa-e9877-646885f67d-vwf6d.18c102807be75c6b namespace: kserve-ci-e2e-test resourceVersion: "17002" uid: 5f1e4863-fda1-4ce1-9371-5481f5df1fa6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T18:54:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-e9877} kind: Pod name: model-chainer-raw-hpa-e9877-646885f67d-vwf6d namespace: kserve-ci-e2e-test resourceVersion: "16883" uid: 91eea447-c8b2-4279-8ec6-5c0583dba188 kind: Event lastTimestamp: "2026-07-10T18:55:05Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-10T18:54:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T18:55:05Z" name: model-chainer-raw-hpa-e9877-646885f67d-vwf6d.18c10280f3bbc544 namespace: kserve-ci-e2e-test resourceVersion: "17357" uid: 7ee5be5a-e2c5-478a-b8d0-a83c1f6ab2a1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-e9877-646885f67d namespace: kserve-ci-e2e-test resourceVersion: "16876" uid: e0b20dc0-ab94-4c59-a8c4-9b05bd6ecb06 kind: Event lastTimestamp: "2026-07-10T18:54:28Z" message: 'Created pod: model-chainer-raw-hpa-e9877-646885f67d-vwf6d' metadata: creationTimestamp: "2026-07-10T18:54:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:54:28Z" name: model-chainer-raw-hpa-e9877-646885f67d.18c1027e25219a34 namespace: kserve-ci-e2e-test resourceVersion: "16882" uid: 6f43919f-e63e-41d8-93d8-e2d79455dae7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-e9877 namespace: kserve-ci-e2e-test resourceVersion: "16875" uid: e5bfa964-d100-48b1-a30b-aaffa15e12e2 kind: Event lastTimestamp: "2026-07-10T18:54:28Z" message: Scaled up replica set model-chainer-raw-hpa-e9877-646885f67d from 0 to 1 metadata: creationTimestamp: "2026-07-10T18:54:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T18:54:28Z" name: model-chainer-raw-hpa-e9877.18c1027e23005ea0 namespace: kserve-ci-e2e-test resourceVersion: "16878" uid: 71198e03-9018-432e-b850-438a87072cb9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T18:54:28Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-e9877 namespace: kserve-ci-e2e-test resourceVersion: "16895" uid: e8bb98ab-f22b-4ea9-92fe-25f7de42714a kind: Event lastTimestamp: "2026-07-10T18:54:28Z" message: InferenceGraph [model-chainer-raw-hpa-e9877] is Ready metadata: creationTimestamp: "2026-07-10T18:54:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T18:54:28Z" name: model-chainer-raw-hpa-e9877.18c1027e2e050c8d namespace: kserve-ci-e2e-test resourceVersion: "16897" uid: 04aac428-1b9e-418d-9568-0a214d529574 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:00:48.569657Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9 namespace: kserve-ci-e2e-test resourceVersion: "20221" uid: b96f75d2-dc23-4cdc-baca-809892dedc44 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-3fcc6-predictor-6b698db54-qskc9 to ip-10-0-134-57.ec2.internal metadata: creationTimestamp: "2026-07-10T19:00:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:00:48Z" name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9.18c102d6b34a37f9 namespace: kserve-ci-e2e-test resourceVersion: "20226" uid: 7cb96a4f-0de8-488d-9ad7-0cd067900c88 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-856b4cf847-czcjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:49Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9 namespace: kserve-ci-e2e-test resourceVersion: "20232" uid: b96f75d2-dc23-4cdc-baca-809892dedc44 kind: Event lastTimestamp: "2026-07-10T19:00:49Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:00:49Z" name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9.18c102d6cdb99bf6 namespace: kserve-ci-e2e-test resourceVersion: "20240" uid: d0577209-4728-4000-9892-3fa7330b8409 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9 namespace: kserve-ci-e2e-test resourceVersion: "20223" uid: b96f75d2-dc23-4cdc-baca-809892dedc44 kind: Event lastTimestamp: "2026-07-10T19:00:49Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:e7d18b8520466922c73925c5601004b31b3c3ec90f89a0821d13bdeee1848812" already present on machine metadata: creationTimestamp: "2026-07-10T19:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:00:49Z" name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9.18c102d6cf87de92 namespace: kserve-ci-e2e-test resourceVersion: "20242" uid: 5f7624b9-d5f5-4103-84fb-823d92d51f9d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9 namespace: kserve-ci-e2e-test resourceVersion: "20223" uid: b96f75d2-dc23-4cdc-baca-809892dedc44 kind: Event lastTimestamp: "2026-07-10T19:00:49Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:00:49Z" name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9.18c102d6d3ea75bb namespace: kserve-ci-e2e-test resourceVersion: "20244" uid: e590ffc2-0e32-48ce-bfcd-ee0c8c067da3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9 namespace: kserve-ci-e2e-test resourceVersion: "20223" uid: b96f75d2-dc23-4cdc-baca-809892dedc44 kind: Event lastTimestamp: "2026-07-10T19:00:49Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:00:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:00:49Z" name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9.18c102d6d463fc52 namespace: kserve-ci-e2e-test resourceVersion: "20245" uid: e3747c2e-8ea8-4ef5-afd4-8cb635492217 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9 namespace: kserve-ci-e2e-test resourceVersion: "20223" uid: b96f75d2-dc23-4cdc-baca-809892dedc44 kind: Event lastTimestamp: "2026-07-10T19:00:52Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1724" already present on machine metadata: creationTimestamp: "2026-07-10T19:00:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:00:52Z" name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9.18c102d7bae156c9 namespace: kserve-ci-e2e-test resourceVersion: "20285" uid: 053a3ceb-5d8c-4125-a559-263bbafcc2bf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9 namespace: kserve-ci-e2e-test resourceVersion: "20223" uid: b96f75d2-dc23-4cdc-baca-809892dedc44 kind: Event lastTimestamp: "2026-07-10T19:00:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:00:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:00:53Z" name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9.18c102d7bdd57760 namespace: kserve-ci-e2e-test resourceVersion: "20287" uid: c7857b5e-af94-463f-bb26-0bdf59e7df72 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9 namespace: kserve-ci-e2e-test resourceVersion: "20223" uid: b96f75d2-dc23-4cdc-baca-809892dedc44 kind: Event lastTimestamp: "2026-07-10T19:00:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:00:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:00:53Z" name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9.18c102d7be523d87 namespace: kserve-ci-e2e-test resourceVersion: "20288" uid: 2e02d240-a5f8-415e-8831-37733538ad02 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9 namespace: kserve-ci-e2e-test resourceVersion: "20223" uid: b96f75d2-dc23-4cdc-baca-809892dedc44 kind: Event lastTimestamp: "2026-07-10T19:00:53Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:00:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:00:53Z" name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9.18c102d7be5e4c9a namespace: kserve-ci-e2e-test resourceVersion: "20289" uid: b4a781fd-8538-4d22-8199-6026ff04e696 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9 namespace: kserve-ci-e2e-test resourceVersion: "20223" uid: b96f75d2-dc23-4cdc-baca-809892dedc44 kind: Event lastTimestamp: "2026-07-10T19:00:53Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:00:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:00:53Z" name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9.18c102d7c14e0846 namespace: kserve-ci-e2e-test resourceVersion: "20291" uid: f0f10c7f-0017-48e6-aaed-dd2267ca8c30 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9 namespace: kserve-ci-e2e-test resourceVersion: "20223" uid: b96f75d2-dc23-4cdc-baca-809892dedc44 kind: Event lastTimestamp: "2026-07-10T19:00:53Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:00:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:00:53Z" name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9.18c102d7c1c9de94 namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: 021e0411-7389-4802-86c8-9a8c2e1c49a9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-10T19:00:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9 namespace: kserve-ci-e2e-test resourceVersion: "20223" uid: b96f75d2-dc23-4cdc-baca-809892dedc44 kind: Event lastTimestamp: "2026-07-10T19:02:11Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:00:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:02:11Z" name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9.18c102d832b7cf09 namespace: kserve-ci-e2e-test resourceVersion: "20893" uid: 61d435eb-caa2-48f2-a50b-4da8fb3c70c8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:02:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9 namespace: kserve-ci-e2e-test resourceVersion: "20223" uid: b96f75d2-dc23-4cdc-baca-809892dedc44 kind: Event lastTimestamp: "2026-07-10T19:02:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:02:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:02:08Z" name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9.18c102e969f8a216 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: 296d91c4-e2e6-4880-a2d9-5730a4bc633b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:02:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9 namespace: kserve-ci-e2e-test resourceVersion: "20223" uid: b96f75d2-dc23-4cdc-baca-809892dedc44 kind: Event lastTimestamp: "2026-07-10T19:02:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:02:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:02:08Z" name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9.18c102e969f929ea namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 2d936bfe-3ee7-4b8e-a18f-8f6b971c05ae reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:02:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9 namespace: kserve-ci-e2e-test resourceVersion: "20223" uid: b96f75d2-dc23-4cdc-baca-809892dedc44 kind: Event lastTimestamp: "2026-07-10T19:02:11Z" message: 'Readiness probe failed: Get "https://10.132.0.34:8643/healthz": dial tcp 10.132.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:02:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:02:11Z" name: raw-sklearn-3fcc6-predictor-6b698db54-qskc9.18c102e9e4e081ff namespace: kserve-ci-e2e-test resourceVersion: "20892" uid: b5b335d6-6ef3-494a-8020-8974b33df2b1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-3fcc6-predictor-6b698db54 namespace: kserve-ci-e2e-test resourceVersion: "20210" uid: 4d0b1f3f-412b-4174-889d-10ab4287c991 kind: Event lastTimestamp: "2026-07-10T19:00:48Z" message: 'Created pod: raw-sklearn-3fcc6-predictor-6b698db54-qskc9' metadata: creationTimestamp: "2026-07-10T19:00:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:00:48Z" name: raw-sklearn-3fcc6-predictor-6b698db54.18c102d6b2ccf26b namespace: kserve-ci-e2e-test resourceVersion: "20222" uid: 6adaa409-2d5c-4c2a-9332-9755b8daffc5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-3fcc6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20207" uid: 6a93e85c-c566-4a91-abe0-94dd704f7657 kind: Event lastTimestamp: "2026-07-10T19:00:48Z" message: Scaled up replica set raw-sklearn-3fcc6-predictor-6b698db54 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:00:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:00:48Z" name: raw-sklearn-3fcc6-predictor.18c102d6af4883b6 namespace: kserve-ci-e2e-test resourceVersion: "20212" uid: 4881b686-7f3a-48aa-8a25-6727b9b28ef2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:01:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-3fcc6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20209" uid: 651c3bf6-3f67-4c10-b061-b6ceae870785 kind: Event lastTimestamp: "2026-07-10T19:01:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:01:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:01:03Z" name: raw-sklearn-3fcc6-predictor.18c102da2e6e8a35 namespace: kserve-ci-e2e-test resourceVersion: "20362" uid: b36333be-2132-4399-825b-ad11bb563ef4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:01:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-3fcc6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20209" uid: 651c3bf6-3f67-4c10-b061-b6ceae870785 kind: Event lastTimestamp: "2026-07-10T19:01:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:01:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:01:03Z" name: raw-sklearn-3fcc6-predictor.18c102da2ef3b5d8 namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: 476f47a3-47b0-41f9-ab55-7aed7605ccfc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:01:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-3fcc6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20361" uid: 651c3bf6-3f67-4c10-b061-b6ceae870785 kind: Event lastTimestamp: "2026-07-10T19:02:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:01:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:02:03Z" name: raw-sklearn-3fcc6-predictor.18c102ddade0d2ea namespace: kserve-ci-e2e-test resourceVersion: "20771" uid: 9399e613-e42d-466f-af18-940d7a596d58 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:01:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-3fcc6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20361" uid: 651c3bf6-3f67-4c10-b061-b6ceae870785 kind: Event lastTimestamp: "2026-07-10T19:02:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:01:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:02:03Z" name: raw-sklearn-3fcc6-predictor.18c102ddae63b2af namespace: kserve-ci-e2e-test resourceVersion: "20772" uid: e50077ce-107b-4d71-9e28-21707d4a4299 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-10T19:00:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-3fcc6 namespace: kserve-ci-e2e-test resourceVersion: "20200" uid: 68f83267-bed9-4936-bebf-373e6c9ceb1b kind: Event lastTimestamp: "2026-07-10T19:02:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:00:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:02:01Z" name: raw-sklearn-3fcc6.18c102d6af19b07b namespace: kserve-ci-e2e-test resourceVersion: "20755" uid: f85d85a9-1257-461e-899b-a2384abd2fb5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:00:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-3fcc6 namespace: kserve-ci-e2e-test resourceVersion: "20200" uid: 68f83267-bed9-4936-bebf-373e6c9ceb1b kind: Event lastTimestamp: "2026-07-10T19:00:48Z" message: 'Failed to update status for InferenceService "raw-sklearn-3fcc6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-3fcc6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:00:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:00:48Z" name: raw-sklearn-3fcc6.18c102d6af82be3e namespace: kserve-ci-e2e-test resourceVersion: "20214" uid: 04d780fe-ae50-4f02-836a-7a771e866def reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:02:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-3fcc6 namespace: kserve-ci-e2e-test resourceVersion: "20752" uid: 68f83267-bed9-4936-bebf-373e6c9ceb1b kind: Event lastTimestamp: "2026-07-10T19:02:01Z" message: InferenceService [raw-sklearn-3fcc6] is Ready metadata: creationTimestamp: "2026-07-10T19:02:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:02:01Z" name: raw-sklearn-3fcc6.18c102e79a07124c namespace: kserve-ci-e2e-test resourceVersion: "20754" uid: 1cc619a9-0d08-4a35-b516-61eab73d272e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:02:08.834592Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8 namespace: kserve-ci-e2e-test resourceVersion: "20831" uid: 79f910ba-aeaf-4585-b88a-dbd66d88bb64 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8 to ip-10-0-134-57.ec2.internal metadata: creationTimestamp: "2026-07-10T19:02:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:02:08Z" name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8.18c102e96373ddcf namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: 6621d6f5-1c14-4b1f-8b56-305d8aec1a93 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-856b4cf847-czcjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:02:09Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8 namespace: kserve-ci-e2e-test resourceVersion: "20851" uid: 79f910ba-aeaf-4585-b88a-dbd66d88bb64 kind: Event lastTimestamp: "2026-07-10T19:02:09Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:02:09Z" name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8.18c102e97dbeb67f namespace: kserve-ci-e2e-test resourceVersion: "20868" uid: 3d298834-4e3a-4ae6-8d7b-0d178f6a033d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:02:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8 namespace: kserve-ci-e2e-test resourceVersion: "20835" uid: 79f910ba-aeaf-4585-b88a-dbd66d88bb64 kind: Event lastTimestamp: "2026-07-10T19:02:09Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:e7d18b8520466922c73925c5601004b31b3c3ec90f89a0821d13bdeee1848812" already present on machine metadata: creationTimestamp: "2026-07-10T19:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:02:09Z" name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8.18c102e97fa510de namespace: kserve-ci-e2e-test resourceVersion: "20870" uid: 55664a5e-ce4f-4be2-bc06-73e74b212a39 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:02:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8 namespace: kserve-ci-e2e-test resourceVersion: "20835" uid: 79f910ba-aeaf-4585-b88a-dbd66d88bb64 kind: Event lastTimestamp: "2026-07-10T19:02:09Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:02:09Z" name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8.18c102e9826aee5e namespace: kserve-ci-e2e-test resourceVersion: "20871" uid: 35dbae51-f532-49a9-9b61-074f367c3a41 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:02:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8 namespace: kserve-ci-e2e-test resourceVersion: "20835" uid: 79f910ba-aeaf-4585-b88a-dbd66d88bb64 kind: Event lastTimestamp: "2026-07-10T19:02:09Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:02:09Z" name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8.18c102e982e7ed56 namespace: kserve-ci-e2e-test resourceVersion: "20872" uid: aa460a4e-14d6-4259-8bce-627f14e04b1c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:02:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8 namespace: kserve-ci-e2e-test resourceVersion: "20835" uid: 79f910ba-aeaf-4585-b88a-dbd66d88bb64 kind: Event lastTimestamp: "2026-07-10T19:02:13Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1724" already present on machine metadata: creationTimestamp: "2026-07-10T19:02:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:02:13Z" name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8.18c102ea700a89f8 namespace: kserve-ci-e2e-test resourceVersion: "20907" uid: 729ec7af-408b-4d68-99f2-6fd4359da8a9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:02:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8 namespace: kserve-ci-e2e-test resourceVersion: "20835" uid: 79f910ba-aeaf-4585-b88a-dbd66d88bb64 kind: Event lastTimestamp: "2026-07-10T19:02:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:02:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:02:13Z" name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8.18c102ea731f62e8 namespace: kserve-ci-e2e-test resourceVersion: "20909" uid: d435efe0-cefb-409e-aaa7-14033544e0af reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:02:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8 namespace: kserve-ci-e2e-test resourceVersion: "20835" uid: 79f910ba-aeaf-4585-b88a-dbd66d88bb64 kind: Event lastTimestamp: "2026-07-10T19:02:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:02:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:02:13Z" name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8.18c102ea739fc385 namespace: kserve-ci-e2e-test resourceVersion: "20910" uid: f6be9ad8-7a72-437d-a272-7bbbdfb88b5e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:02:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8 namespace: kserve-ci-e2e-test resourceVersion: "20835" uid: 79f910ba-aeaf-4585-b88a-dbd66d88bb64 kind: Event lastTimestamp: "2026-07-10T19:02:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:02:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:02:13Z" name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8.18c102ea73ac99b6 namespace: kserve-ci-e2e-test resourceVersion: "20911" uid: 9528f895-e09c-4d78-a675-04467093e210 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:02:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8 namespace: kserve-ci-e2e-test resourceVersion: "20835" uid: 79f910ba-aeaf-4585-b88a-dbd66d88bb64 kind: Event lastTimestamp: "2026-07-10T19:02:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:02:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:02:13Z" name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8.18c102ea77217f34 namespace: kserve-ci-e2e-test resourceVersion: "20914" uid: d1a28e9a-8550-4e1f-b47a-ab05e2c45b26 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:02:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8 namespace: kserve-ci-e2e-test resourceVersion: "20835" uid: 79f910ba-aeaf-4585-b88a-dbd66d88bb64 kind: Event lastTimestamp: "2026-07-10T19:02:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:02:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:02:13Z" name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8.18c102ea77a0e70a namespace: kserve-ci-e2e-test resourceVersion: "20915" uid: 99a480b6-92d4-4e94-ba41-7eb29f385838 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-10T19:02:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8 namespace: kserve-ci-e2e-test resourceVersion: "20835" uid: 79f910ba-aeaf-4585-b88a-dbd66d88bb64 kind: Event lastTimestamp: "2026-07-10T19:03:30Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:02:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:03:30Z" name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8.18c102eaac1a7d65 namespace: kserve-ci-e2e-test resourceVersion: "21529" uid: 80f57b23-7d98-44c5-88be-5be2d1bf91e6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:03:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8 namespace: kserve-ci-e2e-test resourceVersion: "20835" uid: 79f910ba-aeaf-4585-b88a-dbd66d88bb64 kind: Event lastTimestamp: "2026-07-10T19:03:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:03:28Z" name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8.18c102fc0a621f95 namespace: kserve-ci-e2e-test resourceVersion: "21485" uid: 3dc5d844-fe36-441a-a0c2-ee696e293999 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:03:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8 namespace: kserve-ci-e2e-test resourceVersion: "20835" uid: 79f910ba-aeaf-4585-b88a-dbd66d88bb64 kind: Event lastTimestamp: "2026-07-10T19:03:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:03:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:03:28Z" name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8.18c102fc0a675bf9 namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: d8c64d4c-2b40-4c12-a0c8-e2a3d7ab4685 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:03:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8 namespace: kserve-ci-e2e-test resourceVersion: "20835" uid: 79f910ba-aeaf-4585-b88a-dbd66d88bb64 kind: Event lastTimestamp: "2026-07-10T19:03:30Z" message: 'Readiness probe failed: Get "https://10.132.0.35:8643/healthz": dial tcp 10.132.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:03:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:03:30Z" name: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8.18c102fc5e65c11b namespace: kserve-ci-e2e-test resourceVersion: "21528" uid: a0d2e98c-f2ae-4d7c-a5e2-9d5c6121ca46 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-57.ec2.internal source: component: kubelet host: ip-10-0-134-57.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:02:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-6e4de-predictor-66cd464d98 namespace: kserve-ci-e2e-test resourceVersion: "20823" uid: 17aa44a0-1238-4655-9f0d-48c02c8a71f2 kind: Event lastTimestamp: "2026-07-10T19:02:08Z" message: 'Created pod: raw-sklearn-runtime-6e4de-predictor-66cd464d98-klwg8' metadata: creationTimestamp: "2026-07-10T19:02:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:02:08Z" name: raw-sklearn-runtime-6e4de-predictor-66cd464d98.18c102e9624a202c namespace: kserve-ci-e2e-test resourceVersion: "20833" uid: af0ad24f-37f8-4d57-9a60-20f7f924949a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:02:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-6e4de-predictor namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: 559202a3-3ed0-4314-a8f3-340a38b8d548 kind: Event lastTimestamp: "2026-07-10T19:02:08Z" message: Scaled up replica set raw-sklearn-runtime-6e4de-predictor-66cd464d98 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:02:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:02:08Z" name: raw-sklearn-runtime-6e4de-predictor.18c102e95e34c00a namespace: kserve-ci-e2e-test resourceVersion: "20825" uid: 74282258-a731-490e-847f-e8eb3aed85e8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:02:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-6e4de-predictor namespace: kserve-ci-e2e-test resourceVersion: "20822" uid: 10e6ddb8-e46c-4348-9213-03caede21299 kind: Event lastTimestamp: "2026-07-10T19:02:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:02:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:02:38Z" name: raw-sklearn-runtime-6e4de-predictor.18c102ecdd02f8bc namespace: kserve-ci-e2e-test resourceVersion: "21107" uid: 3331d459-005b-4977-8203-11549130aaa7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:02:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-6e4de-predictor namespace: kserve-ci-e2e-test resourceVersion: "20822" uid: 10e6ddb8-e46c-4348-9213-03caede21299 kind: Event lastTimestamp: "2026-07-10T19:02:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:02:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:02:38Z" name: raw-sklearn-runtime-6e4de-predictor.18c102ecddc3dbc0 namespace: kserve-ci-e2e-test resourceVersion: "21108" uid: b33e03ba-af19-4cd8-9295-199857185da0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:02:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-6e4de-predictor namespace: kserve-ci-e2e-test resourceVersion: "20994" uid: 10e6ddb8-e46c-4348-9213-03caede21299 kind: Event lastTimestamp: "2026-07-10T19:03:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:02:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:03:23Z" name: raw-sklearn-runtime-6e4de-predictor.18c102f3dbe80aca namespace: kserve-ci-e2e-test resourceVersion: "21434" uid: 91722b71-0953-4c13-8731-387731ea93bb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:02:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-6e4de-predictor namespace: kserve-ci-e2e-test resourceVersion: "20994" uid: 10e6ddb8-e46c-4348-9213-03caede21299 kind: Event lastTimestamp: "2026-07-10T19:03:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:02:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:03:23Z" name: raw-sklearn-runtime-6e4de-predictor.18c102f3dc8447cf namespace: kserve-ci-e2e-test resourceVersion: "21435" uid: 2dc97741-5687-461e-ac05-725219afba11 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-10T19:02:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-6e4de namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: c69aea21-a230-4685-a2c3-97d10701d07f kind: Event lastTimestamp: "2026-07-10T19:03:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:02:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:03:20Z" name: raw-sklearn-runtime-6e4de.18c102e95dcd801b namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: 6edcc285-9e0a-4b6b-9574-da6b3b792c29 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:02:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-6e4de namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: c69aea21-a230-4685-a2c3-97d10701d07f kind: Event lastTimestamp: "2026-07-10T19:02:08Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-6e4de": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-6e4de": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:02:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:02:08Z" name: raw-sklearn-runtime-6e4de.18c102e95e56cb33 namespace: kserve-ci-e2e-test resourceVersion: "20826" uid: 7d6ddfd7-7009-48f9-b3cb-a69974a8aff6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:03:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-6e4de namespace: kserve-ci-e2e-test resourceVersion: "21402" uid: c69aea21-a230-4685-a2c3-97d10701d07f kind: Event lastTimestamp: "2026-07-10T19:03:20Z" message: InferenceService [raw-sklearn-runtime-6e4de] is Ready metadata: creationTimestamp: "2026-07-10T19:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:03:20Z" name: raw-sklearn-runtime-6e4de.18c102fa10876cb9 namespace: kserve-ci-e2e-test resourceVersion: "21404" uid: c72433db-a211-471c-859b-6cf5b4a9dbd7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "21694"