--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-25T00:15:21.258249Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-8448e3-predictor-84676484d9-mvhgn namespace: kserve-ci-e2e-test resourceVersion: "20163" uid: f0339805-0726-40e0-829f-34ba5030726d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-8448e3-predictor-84676484d9-mvhgn to ip-10-0-132-64.ec2.internal metadata: creationTimestamp: "2026-04-25T00:15:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-25T00:15:21Z" name: isvc-init-fail-8448e3-predictor-84676484d9-mvhgn.18a97151226d70e5 namespace: kserve-ci-e2e-test resourceVersion: "20169" uid: 9d6a19f9-7cef-463a-88a1-d7a1a220c86c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56c7745b65-967mt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:21Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-8448e3-predictor-84676484d9-mvhgn namespace: kserve-ci-e2e-test resourceVersion: "20173" uid: f0339805-0726-40e0-829f-34ba5030726d kind: Event lastTimestamp: "2026-04-25T00:15:21Z" message: Add eth0 [10.134.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-25T00:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-25T00:15:21Z" name: isvc-init-fail-8448e3-predictor-84676484d9-mvhgn.18a971513caa34eb namespace: kserve-ci-e2e-test resourceVersion: "20182" uid: fb6c9680-2dbf-4621-916f-67305c903b2c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-25T00:15:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-8448e3-predictor-84676484d9-mvhgn namespace: kserve-ci-e2e-test resourceVersion: "20165" uid: f0339805-0726-40e0-829f-34ba5030726d kind: Event lastTimestamp: "2026-04-25T00:15:26Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:72e9a53b4f6609f050c864ccf8bba3bc53b15bbbb0a09e5de74a3bdc7465dd84" already present on machine metadata: creationTimestamp: "2026-04-25T00:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:15:26Z" name: isvc-init-fail-8448e3-predictor-84676484d9-mvhgn.18a971513dfc369d namespace: kserve-ci-e2e-test resourceVersion: "20227" uid: 37c9fc01-d0a9-4612-b8d1-a730e65f23d6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-25T00:15:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-8448e3-predictor-84676484d9-mvhgn namespace: kserve-ci-e2e-test resourceVersion: "20165" uid: f0339805-0726-40e0-829f-34ba5030726d kind: Event lastTimestamp: "2026-04-25T00:15:26Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-25T00:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:15:26Z" name: isvc-init-fail-8448e3-predictor-84676484d9-mvhgn.18a9715140f07e41 namespace: kserve-ci-e2e-test resourceVersion: "20231" uid: b9ae0243-e7f4-41f5-877a-25c0a834fe51 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-25T00:15:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-8448e3-predictor-84676484d9-mvhgn namespace: kserve-ci-e2e-test resourceVersion: "20165" uid: f0339805-0726-40e0-829f-34ba5030726d kind: Event lastTimestamp: "2026-04-25T00:15:26Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-25T00:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:15:26Z" name: isvc-init-fail-8448e3-predictor-84676484d9-mvhgn.18a971514164f931 namespace: kserve-ci-e2e-test resourceVersion: "20234" uid: 2f1e5be9-dafc-4db3-bfb3-1ac095aae66d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-8448e3-predictor-84676484d9-mvhgn namespace: kserve-ci-e2e-test resourceVersion: "20165" uid: f0339805-0726-40e0-829f-34ba5030726d kind: Event lastTimestamp: "2026-04-25T00:15:30Z" message: Back-off restarting failed container storage-initializer in pod isvc-init-fail-8448e3-predictor-84676484d9-mvhgn_kserve-ci-e2e-test(f0339805-0726-40e0-829f-34ba5030726d) metadata: creationTimestamp: "2026-04-25T00:15:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:15:30Z" name: isvc-init-fail-8448e3-predictor-84676484d9-mvhgn.18a9715333182a14 namespace: kserve-ci-e2e-test resourceVersion: "20266" uid: 7ae4d280-337a-4323-a1b7-7987368f594b reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-8448e3-predictor-84676484d9 namespace: kserve-ci-e2e-test resourceVersion: "20154" uid: 8ac4384d-1395-44a5-8a9a-a59dd796d8e7 kind: Event lastTimestamp: "2026-04-25T00:15:21Z" message: 'Created pod: isvc-init-fail-8448e3-predictor-84676484d9-mvhgn' metadata: creationTimestamp: "2026-04-25T00:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:15:21Z" name: isvc-init-fail-8448e3-predictor-84676484d9.18a9715121e89aae namespace: kserve-ci-e2e-test resourceVersion: "20164" uid: 2b802218-f00b-4ad2-8683-1c373646d464 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-8448e3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20151" uid: 2821e01b-8964-47eb-85a7-6a1ec9d2fd50 kind: Event lastTimestamp: "2026-04-25T00:15:21Z" message: Scaled up replica set isvc-init-fail-8448e3-predictor-84676484d9 from 0 to 1 metadata: creationTimestamp: "2026-04-25T00:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:15:21Z" name: isvc-init-fail-8448e3-predictor.18a971511ee04a8b namespace: kserve-ci-e2e-test resourceVersion: "20156" uid: 66f01755-6f22-4853-93b4-c89fbd0f405f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-25T00:15:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-8448e3 namespace: kserve-ci-e2e-test resourceVersion: "20127" uid: 9c5ff6e9-4a26-43b0-97ad-a2d90c39a5c9 kind: Event lastTimestamp: "2026-04-25T00:15:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-25T00:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:15:30Z" name: isvc-init-fail-8448e3.18a971511ebb8032 namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 325cc821-9a3d-44a6-b34b-d59ff13feedb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-8448e3 namespace: kserve-ci-e2e-test resourceVersion: "20127" uid: 9c5ff6e9-4a26-43b0-97ad-a2d90c39a5c9 kind: Event lastTimestamp: "2026-04-25T00:15:21Z" message: 'Failed to update status for InferenceService "isvc-init-fail-8448e3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-8448e3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-25T00:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:15:21Z" name: isvc-init-fail-8448e3.18a971511f0c62f9 namespace: kserve-ci-e2e-test resourceVersion: "20158" uid: 7835c7c7-1d6e-4c52-86f7-e31e755c0986 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-8448e3 namespace: kserve-ci-e2e-test resourceVersion: "20127" uid: 9c5ff6e9-4a26-43b0-97ad-a2d90c39a5c9 kind: Event lastTimestamp: "2026-04-25T00:15:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-8448e3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-25T00:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:15:21Z" name: isvc-init-fail-8448e3.18a971511f0cb5e8 namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: e3d6c746-e291-48e8-a199-be4fe9b854ce reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-25T00:09:39.332928Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k namespace: kserve-ci-e2e-test resourceVersion: "17317" uid: 6306d78c-102c-40cb-8924-588efa962ea2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k to ip-10-0-132-64.ec2.internal metadata: creationTimestamp: "2026-04-25T00:09:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-25T00:09:39Z" name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k.18a9710186170d7f namespace: kserve-ci-e2e-test resourceVersion: "17321" uid: f9b0cbba-4404-4285-b48c-5cd044cd1adb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56c7745b65-967mt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k namespace: kserve-ci-e2e-test resourceVersion: "17319" uid: 6306d78c-102c-40cb-8924-588efa962ea2 kind: Event lastTimestamp: "2026-04-25T00:09:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-logger-raw-b8788-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-25T00:09:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:39Z" name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k.18a9710192f56e85 namespace: kserve-ci-e2e-test resourceVersion: "17332" uid: f3db22ed-3250-4451-9a85-1264132302db reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:40Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k namespace: kserve-ci-e2e-test resourceVersion: "17325" uid: 6306d78c-102c-40cb-8924-588efa962ea2 kind: Event lastTimestamp: "2026-04-25T00:09:40Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-25T00:09:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-25T00:09:40Z" name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k.18a97101c3bccc04 namespace: kserve-ci-e2e-test resourceVersion: "17341" uid: 1cd22835-d8a7-4cb7-9e40-49f6ac9f8bec reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k namespace: kserve-ci-e2e-test resourceVersion: "17319" uid: 6306d78c-102c-40cb-8924-588efa962ea2 kind: Event lastTimestamp: "2026-04-25T00:09:40Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:72e9a53b4f6609f050c864ccf8bba3bc53b15bbbb0a09e5de74a3bdc7465dd84" already present on machine metadata: creationTimestamp: "2026-04-25T00:09:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:40Z" name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k.18a97101c51f1815 namespace: kserve-ci-e2e-test resourceVersion: "17344" uid: 127bb5cc-bcc7-41b6-8962-57a80d274963 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k namespace: kserve-ci-e2e-test resourceVersion: "17319" uid: 6306d78c-102c-40cb-8924-588efa962ea2 kind: Event lastTimestamp: "2026-04-25T00:09:40Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-25T00:09:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:40Z" name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k.18a97101c80cbf38 namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: 7a2cc407-58e3-4952-b9e9-b62c06bd5c33 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k namespace: kserve-ci-e2e-test resourceVersion: "17319" uid: 6306d78c-102c-40cb-8924-588efa962ea2 kind: Event lastTimestamp: "2026-04-25T00:09:40Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-25T00:09:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:40Z" name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k.18a97101c88093d1 namespace: kserve-ci-e2e-test resourceVersion: "17348" uid: cf015ee0-963a-4ddb-b69a-41270f01196b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k namespace: kserve-ci-e2e-test resourceVersion: "17319" uid: 6306d78c-102c-40cb-8924-588efa962ea2 kind: Event lastTimestamp: "2026-04-25T00:09:44Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1435" already present on machine metadata: creationTimestamp: "2026-04-25T00:09:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:44Z" name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k.18a97102cdab3caa namespace: kserve-ci-e2e-test resourceVersion: "17390" uid: 71d6da0c-13fd-4108-9375-c14789078c95 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k namespace: kserve-ci-e2e-test resourceVersion: "17319" uid: 6306d78c-102c-40cb-8924-588efa962ea2 kind: Event lastTimestamp: "2026-04-25T00:09:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-25T00:09:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:44Z" name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k.18a97102d0d5eba2 namespace: kserve-ci-e2e-test resourceVersion: "17394" uid: 65c78a68-8b2f-47bf-93cd-902191deee01 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k namespace: kserve-ci-e2e-test resourceVersion: "17319" uid: 6306d78c-102c-40cb-8924-588efa962ea2 kind: Event lastTimestamp: "2026-04-25T00:09:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-25T00:09:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:44Z" name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k.18a97102d168b0e6 namespace: kserve-ci-e2e-test resourceVersion: "17395" uid: 1c5b29e4-4fe9-4796-a6e4-d63c9c85f763 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k namespace: kserve-ci-e2e-test resourceVersion: "17319" uid: 6306d78c-102c-40cb-8924-588efa962ea2 kind: Event lastTimestamp: "2026-04-25T00:09:44Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-25T00:09:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:44Z" name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k.18a97102d179817e namespace: kserve-ci-e2e-test resourceVersion: "17396" uid: a84955c3-becf-4c3f-9e53-c304eef7f531 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k namespace: kserve-ci-e2e-test resourceVersion: "17319" uid: 6306d78c-102c-40cb-8924-588efa962ea2 kind: Event lastTimestamp: "2026-04-25T00:09:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-25T00:09:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:44Z" name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k.18a97102d45e544b namespace: kserve-ci-e2e-test resourceVersion: "17398" uid: 7c1070e5-abe1-4557-9672-52162e1a94da reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k namespace: kserve-ci-e2e-test resourceVersion: "17319" uid: 6306d78c-102c-40cb-8924-588efa962ea2 kind: Event lastTimestamp: "2026-04-25T00:09:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-25T00:09:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:44Z" name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k.18a97102d51853f1 namespace: kserve-ci-e2e-test resourceVersion: "17399" uid: a372a988-6ec7-4a7a-979d-08ba6af5a812 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k namespace: kserve-ci-e2e-test resourceVersion: "17319" uid: 6306d78c-102c-40cb-8924-588efa962ea2 kind: Event lastTimestamp: "2026-04-25T00:09:44Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:e376846d2aa0b7fb92a16b8c1edcc05bf3c05895534eec2581922448b745f27a" already present on machine metadata: creationTimestamp: "2026-04-25T00:09:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:44Z" name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k.18a97102d52451f2 namespace: kserve-ci-e2e-test resourceVersion: "17400" uid: 4c9a5879-4ccf-4a69-9a7f-95b952d5ba23 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k namespace: kserve-ci-e2e-test resourceVersion: "17319" uid: 6306d78c-102c-40cb-8924-588efa962ea2 kind: Event lastTimestamp: "2026-04-25T00:09:45Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-25T00:09:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:45Z" name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k.18a97102d872aa22 namespace: kserve-ci-e2e-test resourceVersion: "17401" uid: 378be574-13a9-4fcd-bcd8-fbc39e4e32e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k namespace: kserve-ci-e2e-test resourceVersion: "17319" uid: 6306d78c-102c-40cb-8924-588efa962ea2 kind: Event lastTimestamp: "2026-04-25T00:09:45Z" message: Started container agent metadata: creationTimestamp: "2026-04-25T00:09:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:45Z" name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k.18a97102d8f96fa2 namespace: kserve-ci-e2e-test resourceVersion: "17402" uid: 63788b10-b326-4e72-aa4c-ebcb2c9ed6cf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-25T00:09:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k namespace: kserve-ci-e2e-test resourceVersion: "17319" uid: 6306d78c-102c-40cb-8924-588efa962ea2 kind: Event lastTimestamp: "2026-04-25T00:11:22Z" message: 'Readiness probe failed: dial tcp 10.134.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-25T00:09:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:11:22Z" name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k.18a97103099e33f3 namespace: kserve-ci-e2e-test resourceVersion: "18236" uid: 8fd635de-42cd-4c75-8373-33123358f8c6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-25T00:09:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k namespace: kserve-ci-e2e-test resourceVersion: "17319" uid: 6306d78c-102c-40cb-8924-588efa962ea2 kind: Event lastTimestamp: "2026-04-25T00:11:12Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-25T00:09:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:11:12Z" name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k.18a971034562c8a0 namespace: kserve-ci-e2e-test resourceVersion: "18160" uid: c26b354e-fba3-45fd-b88c-76d8d10d0822 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:11:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k namespace: kserve-ci-e2e-test resourceVersion: "17319" uid: 6306d78c-102c-40cb-8924-588efa962ea2 kind: Event lastTimestamp: "2026-04-25T00:11:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-25T00:11:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:11:04Z" name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k.18a971155e17cc75 namespace: kserve-ci-e2e-test resourceVersion: "18035" uid: f73e71db-a94c-4e63-81c2-5f00db5b1129 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:11:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k namespace: kserve-ci-e2e-test resourceVersion: "17319" uid: 6306d78c-102c-40cb-8924-588efa962ea2 kind: Event lastTimestamp: "2026-04-25T00:11:04Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-25T00:11:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:11:04Z" name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k.18a971155e18072b namespace: kserve-ci-e2e-test resourceVersion: "18036" uid: 6ebdcb74-c57f-4937-9f8c-dd0c2a56cdc4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:11:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k namespace: kserve-ci-e2e-test resourceVersion: "17319" uid: 6306d78c-102c-40cb-8924-588efa962ea2 kind: Event lastTimestamp: "2026-04-25T00:11:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-25T00:11:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:11:04Z" name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k.18a971155e187bda namespace: kserve-ci-e2e-test resourceVersion: "18037" uid: 4d864571-c841-41f4-9c48-57cc48aa6114 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-25T00:11:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k namespace: kserve-ci-e2e-test resourceVersion: "17319" uid: 6306d78c-102c-40cb-8924-588efa962ea2 kind: Event lastTimestamp: "2026-04-25T00:11:22Z" message: 'Readiness probe failed: Get "https://10.134.0.42:8643/healthz": dial tcp 10.134.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-25T00:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:11:22Z" name: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k.18a9711621975389 namespace: kserve-ci-e2e-test resourceVersion: "18234" uid: 7577d28d-19c9-4e34-b2e1-50a997f061a2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-b8788-predictor-6cfdc7f55c namespace: kserve-ci-e2e-test resourceVersion: "17306" uid: 7c9f4c0c-88f8-45be-b2e8-9f7d7ec14924 kind: Event lastTimestamp: "2026-04-25T00:09:39Z" message: 'Created pod: isvc-logger-raw-b8788-predictor-6cfdc7f55c-95t5k' metadata: creationTimestamp: "2026-04-25T00:09:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:09:39Z" name: isvc-logger-raw-b8788-predictor-6cfdc7f55c.18a9710185ae65ef namespace: kserve-ci-e2e-test resourceVersion: "17318" uid: 1535840f-868e-4cc3-8335-cc8562465b29 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-b8788-predictor namespace: kserve-ci-e2e-test resourceVersion: "17304" uid: 10352983-08e6-4c37-8df3-5e148950527e kind: Event lastTimestamp: "2026-04-25T00:09:39Z" message: Scaled up replica set isvc-logger-raw-b8788-predictor-6cfdc7f55c from 0 to 1 metadata: creationTimestamp: "2026-04-25T00:09:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:09:39Z" name: isvc-logger-raw-b8788-predictor.18a9710182acb6b6 namespace: kserve-ci-e2e-test resourceVersion: "17309" uid: 3e19fb2a-1a79-4a6c-b3e4-c13e88e7fdf9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-25T00:09:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-b8788-predictor namespace: kserve-ci-e2e-test resourceVersion: "17305" uid: acfc0411-1919-44e2-a242-c8754ab41113 kind: Event lastTimestamp: "2026-04-25T00:10:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-25T00:09:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:10:24Z" name: isvc-logger-raw-b8788-predictor.18a97105015e3803 namespace: kserve-ci-e2e-test resourceVersion: "17700" uid: 17d3a689-f99b-40d2-8f79-0dffca77b4f3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-25T00:09:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-b8788-predictor namespace: kserve-ci-e2e-test resourceVersion: "17305" uid: acfc0411-1919-44e2-a242-c8754ab41113 kind: Event lastTimestamp: "2026-04-25T00:10:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-25T00:09:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:10:24Z" name: isvc-logger-raw-b8788-predictor.18a9710501c1eb26 namespace: kserve-ci-e2e-test resourceVersion: "17701" uid: 2e402368-2eb3-40d9-802e-549eef3eb0f8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-25T00:10:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-b8788-predictor namespace: kserve-ci-e2e-test resourceVersion: "17477" uid: acfc0411-1919-44e2-a242-c8754ab41113 kind: Event lastTimestamp: "2026-04-25T00:10:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-25T00:10:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:10:54Z" name: isvc-logger-raw-b8788-predictor.18a971088077fcbf namespace: kserve-ci-e2e-test resourceVersion: "17924" uid: 01b3190b-0a02-42b0-902d-19f068418426 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-25T00:10:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-b8788-predictor namespace: kserve-ci-e2e-test resourceVersion: "17477" uid: acfc0411-1919-44e2-a242-c8754ab41113 kind: Event lastTimestamp: "2026-04-25T00:10:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-25T00:10:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:10:54Z" name: isvc-logger-raw-b8788-predictor.18a9710880e7c5c3 namespace: kserve-ci-e2e-test resourceVersion: "17925" uid: 48f523bc-5d79-4933-8793-7feb3b00a76a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-25T00:09:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-b8788 namespace: kserve-ci-e2e-test resourceVersion: "17295" uid: a8cd445d-ce83-42c8-be7d-b8ba0bbb7406 kind: Event lastTimestamp: "2026-04-25T00:10:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-25T00:09:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:10:52Z" name: isvc-logger-raw-b8788.18a97101827e9c5b namespace: kserve-ci-e2e-test resourceVersion: "17912" uid: 14ad5f31-962f-486b-b047-d0fa289823b7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-b8788 namespace: kserve-ci-e2e-test resourceVersion: "17295" uid: a8cd445d-ce83-42c8-be7d-b8ba0bbb7406 kind: Event lastTimestamp: "2026-04-25T00:09:39Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-b8788": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-b8788": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-25T00:09:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:09:39Z" name: isvc-logger-raw-b8788.18a9710182f917b7 namespace: kserve-ci-e2e-test resourceVersion: "17311" uid: 0a3de0d5-6829-44ab-afe1-2c52b6cf2f66 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:10:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-b8788 namespace: kserve-ci-e2e-test resourceVersion: "17910" uid: a8cd445d-ce83-42c8-be7d-b8ba0bbb7406 kind: Event lastTimestamp: "2026-04-25T00:10:52Z" message: InferenceService [isvc-logger-raw-b8788] is Ready metadata: creationTimestamp: "2026-04-25T00:10:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:10:52Z" name: isvc-logger-raw-b8788.18a97112a9d8b0e7 namespace: kserve-ci-e2e-test resourceVersion: "17911" uid: 4efbc669-1bce-401a-bb34-25034beec248 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-25T00:13:34.894134Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw namespace: kserve-ci-e2e-test resourceVersion: "19224" uid: cbff0fd0-0ea3-4820-b252-cc6b5cfd4216 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw to ip-10-0-132-64.ec2.internal metadata: creationTimestamp: "2026-04-25T00:13:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-25T00:13:34Z" name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw.18a971385ea1ca58 namespace: kserve-ci-e2e-test resourceVersion: "19229" uid: 32cfa322-8fff-43f6-9bc0-00ce72cd92f6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56c7745b65-967mt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:13:35Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw namespace: kserve-ci-e2e-test resourceVersion: "19232" uid: cbff0fd0-0ea3-4820-b252-cc6b5cfd4216 kind: Event lastTimestamp: "2026-04-25T00:13:35Z" message: Add eth0 [10.134.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-25T00:13:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-25T00:13:35Z" name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw.18a9713877ee740a namespace: kserve-ci-e2e-test resourceVersion: "19240" uid: b726ba48-42a8-48f0-8830-03585641e301 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:13:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw namespace: kserve-ci-e2e-test resourceVersion: "19226" uid: cbff0fd0-0ea3-4820-b252-cc6b5cfd4216 kind: Event lastTimestamp: "2026-04-25T00:13:35Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:72e9a53b4f6609f050c864ccf8bba3bc53b15bbbb0a09e5de74a3bdc7465dd84" already present on machine metadata: creationTimestamp: "2026-04-25T00:13:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:13:35Z" name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw.18a97138793835a5 namespace: kserve-ci-e2e-test resourceVersion: "19242" uid: eaf63861-7bd5-4902-9546-4af6af43171d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:13:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw namespace: kserve-ci-e2e-test resourceVersion: "19226" uid: cbff0fd0-0ea3-4820-b252-cc6b5cfd4216 kind: Event lastTimestamp: "2026-04-25T00:13:35Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-25T00:13:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:13:35Z" name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw.18a971387bff86af namespace: kserve-ci-e2e-test resourceVersion: "19243" uid: bc70ede5-ab9d-4ab0-b986-f689cf96a33c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:13:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw namespace: kserve-ci-e2e-test resourceVersion: "19226" uid: cbff0fd0-0ea3-4820-b252-cc6b5cfd4216 kind: Event lastTimestamp: "2026-04-25T00:13:35Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-25T00:13:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:13:35Z" name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw.18a971387cb14b20 namespace: kserve-ci-e2e-test resourceVersion: "19245" uid: a95936d4-e8ae-4e52-ba1a-54ed3504afb6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:13:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw namespace: kserve-ci-e2e-test resourceVersion: "19226" uid: cbff0fd0-0ea3-4820-b252-cc6b5cfd4216 kind: Event lastTimestamp: "2026-04-25T00:13:39Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1435" already present on machine metadata: creationTimestamp: "2026-04-25T00:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:13:39Z" name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw.18a971397c6e3937 namespace: kserve-ci-e2e-test resourceVersion: "19285" uid: 1256d977-ce59-4a5d-8328-cb0a2a536c56 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:13:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw namespace: kserve-ci-e2e-test resourceVersion: "19226" uid: cbff0fd0-0ea3-4820-b252-cc6b5cfd4216 kind: Event lastTimestamp: "2026-04-25T00:13:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-25T00:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:13:39Z" name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw.18a971397f856245 namespace: kserve-ci-e2e-test resourceVersion: "19289" uid: 6931e617-19c9-4fc4-acfa-7ecb3b945f5c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:13:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw namespace: kserve-ci-e2e-test resourceVersion: "19226" uid: cbff0fd0-0ea3-4820-b252-cc6b5cfd4216 kind: Event lastTimestamp: "2026-04-25T00:13:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-25T00:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:13:39Z" name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw.18a971397ff815c4 namespace: kserve-ci-e2e-test resourceVersion: "19290" uid: 9a2e459b-9ef1-4348-8067-6397e98c1cb6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:13:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw namespace: kserve-ci-e2e-test resourceVersion: "19226" uid: cbff0fd0-0ea3-4820-b252-cc6b5cfd4216 kind: Event lastTimestamp: "2026-04-25T00:13:39Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-25T00:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:13:39Z" name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw.18a9713980055525 namespace: kserve-ci-e2e-test resourceVersion: "19291" uid: 02e47709-9f00-4100-8cb9-d4d18aad1010 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:13:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw namespace: kserve-ci-e2e-test resourceVersion: "19226" uid: cbff0fd0-0ea3-4820-b252-cc6b5cfd4216 kind: Event lastTimestamp: "2026-04-25T00:13:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-25T00:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:13:39Z" name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw.18a9713982bb5f62 namespace: kserve-ci-e2e-test resourceVersion: "19293" uid: f3f426cb-17ca-4c54-a5a5-f98db1216628 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:13:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw namespace: kserve-ci-e2e-test resourceVersion: "19226" uid: cbff0fd0-0ea3-4820-b252-cc6b5cfd4216 kind: Event lastTimestamp: "2026-04-25T00:13:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-25T00:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:13:39Z" name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw.18a97139831e0a72 namespace: kserve-ci-e2e-test resourceVersion: "19294" uid: 90381666-405c-4c92-bb11-3e21c15a17ae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-25T00:13:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw namespace: kserve-ci-e2e-test resourceVersion: "19226" uid: cbff0fd0-0ea3-4820-b252-cc6b5cfd4216 kind: Event lastTimestamp: "2026-04-25T00:14:47Z" message: 'Readiness probe failed: dial tcp 10.134.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-25T00:13:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:14:47Z" name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw.18a97139f42f14ab namespace: kserve-ci-e2e-test resourceVersion: "19778" uid: 6fad3dba-e005-475c-8ccb-3a3b51ddf210 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw namespace: kserve-ci-e2e-test resourceVersion: "19226" uid: cbff0fd0-0ea3-4820-b252-cc6b5cfd4216 kind: Event lastTimestamp: "2026-04-25T00:15:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-25T00:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:15:21Z" name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw.18a971511d1464b1 namespace: kserve-ci-e2e-test resourceVersion: "20143" uid: bb9a5c85-4774-4f12-9420-d4e76dac4c3b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw namespace: kserve-ci-e2e-test resourceVersion: "19226" uid: cbff0fd0-0ea3-4820-b252-cc6b5cfd4216 kind: Event lastTimestamp: "2026-04-25T00:15:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-25T00:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:15:21Z" name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw.18a971511d14e601 namespace: kserve-ci-e2e-test resourceVersion: "20147" uid: 77c55388-8950-4080-a243-a6e7939d6359 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw namespace: kserve-ci-e2e-test resourceVersion: "19226" uid: cbff0fd0-0ea3-4820-b252-cc6b5cfd4216 kind: Event lastTimestamp: "2026-04-25T00:15:22Z" message: 'Readiness probe failed: Get "https://10.134.0.44:8643/healthz": dial tcp 10.134.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-25T00:15:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:15:22Z" name: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw.18a9715178688577 namespace: kserve-ci-e2e-test resourceVersion: "20203" uid: 1a90e291-4fde-4eb6-b7ca-13d7f2368cc2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:13:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-bbf281-predictor-6b7cf7ddb4 namespace: kserve-ci-e2e-test resourceVersion: "19213" uid: 5000cfab-88da-42ea-96fd-3aced76c3676 kind: Event lastTimestamp: "2026-04-25T00:13:34Z" message: 'Created pod: isvc-primary-bbf281-predictor-6b7cf7ddb4-9mhhw' metadata: creationTimestamp: "2026-04-25T00:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:13:34Z" name: isvc-primary-bbf281-predictor-6b7cf7ddb4.18a971385e17c710 namespace: kserve-ci-e2e-test resourceVersion: "19225" uid: 4bcf0900-50d9-4234-86c0-657cecde4f58 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:13:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-bbf281-predictor namespace: kserve-ci-e2e-test resourceVersion: "19212" uid: f60a9ad4-9503-4a21-b71c-8f17ee0fb713 kind: Event lastTimestamp: "2026-04-25T00:13:34Z" message: Scaled up replica set isvc-primary-bbf281-predictor-6b7cf7ddb4 from 0 to 1 metadata: creationTimestamp: "2026-04-25T00:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:13:34Z" name: isvc-primary-bbf281-predictor.18a97138599286d1 namespace: kserve-ci-e2e-test resourceVersion: "19215" uid: cff3cd91-72c6-4ef0-befb-b1b3b9d6d9b8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-25T00:13:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-bbf281-predictor namespace: kserve-ci-e2e-test resourceVersion: "19214" uid: 0486b4b6-ac3d-4acf-93e6-7eae8338915b kind: Event lastTimestamp: "2026-04-25T00:14:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-25T00:13:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:14:19Z" name: isvc-primary-bbf281-predictor.18a9713bd8fb61b6 namespace: kserve-ci-e2e-test resourceVersion: "19579" uid: 4b40d553-ba6d-451c-a06b-f7f344beb7d4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-25T00:13:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-bbf281-predictor namespace: kserve-ci-e2e-test resourceVersion: "19214" uid: 0486b4b6-ac3d-4acf-93e6-7eae8338915b kind: Event lastTimestamp: "2026-04-25T00:14:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-25T00:13:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:14:19Z" name: isvc-primary-bbf281-predictor.18a9713bd972583f namespace: kserve-ci-e2e-test resourceVersion: "19580" uid: ac7b7bba-0358-46af-9a5e-6a65dfefb8b2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-25T00:14:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-bbf281-predictor namespace: kserve-ci-e2e-test resourceVersion: "19371" uid: 0486b4b6-ac3d-4acf-93e6-7eae8338915b kind: Event lastTimestamp: "2026-04-25T00:15:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-25T00:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:15:19Z" name: isvc-primary-bbf281-predictor.18a97146560bfda6 namespace: kserve-ci-e2e-test resourceVersion: "20084" uid: f712264e-7677-4780-8192-98393bde10e0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-25T00:14:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-bbf281-predictor namespace: kserve-ci-e2e-test resourceVersion: "19371" uid: 0486b4b6-ac3d-4acf-93e6-7eae8338915b kind: Event lastTimestamp: "2026-04-25T00:15:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-25T00:14:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:15:19Z" name: isvc-primary-bbf281-predictor.18a97146566e542b namespace: kserve-ci-e2e-test resourceVersion: "20085" uid: 40b1f728-d3ed-4adf-a186-e71ebd0338b5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-25T00:13:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-bbf281 namespace: kserve-ci-e2e-test resourceVersion: "19185" uid: 7ae14ee2-f5e5-478a-bd72-efdb4fb2824a kind: Event lastTimestamp: "2026-04-25T00:14:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-25T00:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:14:57Z" name: isvc-primary-bbf281.18a9713859acbfd9 namespace: kserve-ci-e2e-test resourceVersion: "19858" uid: c1343f32-8c72-41a7-80dc-32b8d733ba2c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:13:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-bbf281 namespace: kserve-ci-e2e-test resourceVersion: "19185" uid: 7ae14ee2-f5e5-478a-bd72-efdb4fb2824a kind: Event lastTimestamp: "2026-04-25T00:13:34Z" message: 'Failed to update status for InferenceService "isvc-primary-bbf281": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-bbf281": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-25T00:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:13:34Z" name: isvc-primary-bbf281.18a971385a262da0 namespace: kserve-ci-e2e-test resourceVersion: "19218" uid: 9bb6d42b-5944-4d0b-a4c9-0a727b1c4a5b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:13:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-bbf281 namespace: kserve-ci-e2e-test resourceVersion: "19185" uid: 7ae14ee2-f5e5-478a-bd72-efdb4fb2824a kind: Event lastTimestamp: "2026-04-25T00:13:34Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-bbf281": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-25T00:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:13:34Z" name: isvc-primary-bbf281.18a971385a267788 namespace: kserve-ci-e2e-test resourceVersion: "19219" uid: fb64ca0e-ffe6-4e6c-8444-da169c35497f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:14:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-bbf281 namespace: kserve-ci-e2e-test resourceVersion: "19856" uid: 7ae14ee2-f5e5-478a-bd72-efdb4fb2824a kind: Event lastTimestamp: "2026-04-25T00:14:57Z" message: InferenceService [isvc-primary-bbf281] is Ready metadata: creationTimestamp: "2026-04-25T00:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:14:57Z" name: isvc-primary-bbf281.18a9714bab1c2ea5 namespace: kserve-ci-e2e-test resourceVersion: "19857" uid: 0d223969-d3dd-47b9-9dbf-91472dc8c0ff reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-25T00:04:13.547263Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99 namespace: kserve-ci-e2e-test resourceVersion: "14290" uid: fce959c1-497c-4e87-8e24-4b5ef6e69338 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99 to ip-10-0-132-64.ec2.internal metadata: creationTimestamp: "2026-04-25T00:04:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-25T00:04:13Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99.18a970b5abc05bfc namespace: kserve-ci-e2e-test resourceVersion: "14296" uid: 2fa3d89d-d061-43aa-b412-61bf7c6065ad reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56c7745b65-967mt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:04:13Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99 namespace: kserve-ci-e2e-test resourceVersion: "14302" uid: fce959c1-497c-4e87-8e24-4b5ef6e69338 kind: Event lastTimestamp: "2026-04-25T00:04:13Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-25T00:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-25T00:04:13Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99.18a970b5c56c9533 namespace: kserve-ci-e2e-test resourceVersion: "14308" uid: d657571c-71af-4743-9733-5b65c957d2c5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:04:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99 namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: fce959c1-497c-4e87-8e24-4b5ef6e69338 kind: Event lastTimestamp: "2026-04-25T00:04:14Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:72e9a53b4f6609f050c864ccf8bba3bc53b15bbbb0a09e5de74a3bdc7465dd84" metadata: creationTimestamp: "2026-04-25T00:04:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:04:14Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99.18a970b5c6c25eb5 namespace: kserve-ci-e2e-test resourceVersion: "14310" uid: 8dc523c5-a7fc-44f7-8e64-9e754821d90f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:04:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99 namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: fce959c1-497c-4e87-8e24-4b5ef6e69338 kind: Event lastTimestamp: "2026-04-25T00:04:16Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:72e9a53b4f6609f050c864ccf8bba3bc53b15bbbb0a09e5de74a3bdc7465dd84" in 2.881s (2.881s including waiting). Image size: 301492132 bytes.' metadata: creationTimestamp: "2026-04-25T00:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:04:16Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99.18a970b672841c72 namespace: kserve-ci-e2e-test resourceVersion: "14329" uid: a239bb78-cbbd-4ac2-bf01-07a1ff7337f2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:04:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99 namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: fce959c1-497c-4e87-8e24-4b5ef6e69338 kind: Event lastTimestamp: "2026-04-25T00:04:16Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-25T00:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:04:16Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99.18a970b67694fde7 namespace: kserve-ci-e2e-test resourceVersion: "14330" uid: 278672d0-09ec-4836-adf5-7c85db5cd716 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:04:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99 namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: fce959c1-497c-4e87-8e24-4b5ef6e69338 kind: Event lastTimestamp: "2026-04-25T00:04:16Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-25T00:04:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:04:16Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99.18a970b677208970 namespace: kserve-ci-e2e-test resourceVersion: "14331" uid: da7e1568-6619-4b6c-af37-4a8257b03e62 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:04:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99 namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: fce959c1-497c-4e87-8e24-4b5ef6e69338 kind: Event lastTimestamp: "2026-04-25T00:04:21Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1435" metadata: creationTimestamp: "2026-04-25T00:04:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:04:21Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99.18a970b78cf4c45b namespace: kserve-ci-e2e-test resourceVersion: "14372" uid: 2e7043ec-ad63-4585-a137-0853e6f5bda0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:04:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99 namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: fce959c1-497c-4e87-8e24-4b5ef6e69338 kind: Event lastTimestamp: "2026-04-25T00:04:35Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1435" in 13.759s (13.759s including waiting). Image size: 1560924583 bytes.' metadata: creationTimestamp: "2026-04-25T00:04:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:04:35Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99.18a970bac110dc94 namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: 304bf68d-d3aa-4e95-80ed-8866519833a2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:04:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99 namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: fce959c1-497c-4e87-8e24-4b5ef6e69338 kind: Event lastTimestamp: "2026-04-25T00:04:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-25T00:04:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:04:35Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99.18a970bac4c86b4d namespace: kserve-ci-e2e-test resourceVersion: "14475" uid: 8fd93576-02ed-49da-bd04-32a692b0a4d5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:04:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99 namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: fce959c1-497c-4e87-8e24-4b5ef6e69338 kind: Event lastTimestamp: "2026-04-25T00:04:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-25T00:04:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:04:35Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99.18a970bac532fd81 namespace: kserve-ci-e2e-test resourceVersion: "14476" uid: f21e33b4-a484-4232-a39a-c81490ca3fe1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:04:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99 namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: fce959c1-497c-4e87-8e24-4b5ef6e69338 kind: Event lastTimestamp: "2026-04-25T00:04:35Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-25T00:04:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:04:35Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99.18a970bac53eeae0 namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: b92e07b0-9375-415d-8730-172ffdaf5d88 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:04:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99 namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: fce959c1-497c-4e87-8e24-4b5ef6e69338 kind: Event lastTimestamp: "2026-04-25T00:04:37Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.398s (2.398s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-25T00:04:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:04:37Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99.18a970bb542fdc14 namespace: kserve-ci-e2e-test resourceVersion: "14495" uid: 114bcdf6-0eb4-4bbd-8c48-5f258596684f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:04:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99 namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: fce959c1-497c-4e87-8e24-4b5ef6e69338 kind: Event lastTimestamp: "2026-04-25T00:04:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-25T00:04:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:04:37Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99.18a970bb57684bb5 namespace: kserve-ci-e2e-test resourceVersion: "14496" uid: 1779f285-449b-4763-b952-4ff5090b9d75 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:04:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99 namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: fce959c1-497c-4e87-8e24-4b5ef6e69338 kind: Event lastTimestamp: "2026-04-25T00:04:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-25T00:04:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:04:37Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99.18a970bb57d12bd5 namespace: kserve-ci-e2e-test resourceVersion: "14497" uid: 1aa82e53-40c6-4e7e-afd6-4a152be2e61f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:04:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99 namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: fce959c1-497c-4e87-8e24-4b5ef6e69338 kind: Event lastTimestamp: "2026-04-25T00:04:37Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:e376846d2aa0b7fb92a16b8c1edcc05bf3c05895534eec2581922448b745f27a" metadata: creationTimestamp: "2026-04-25T00:04:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:04:37Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99.18a970bb57de1741 namespace: kserve-ci-e2e-test resourceVersion: "14498" uid: 3062345e-8d59-4aea-9a6e-b63c36eaae90 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:04:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99 namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: fce959c1-497c-4e87-8e24-4b5ef6e69338 kind: Event lastTimestamp: "2026-04-25T00:04:40Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:e376846d2aa0b7fb92a16b8c1edcc05bf3c05895534eec2581922448b745f27a" in 2.576s (2.576s including waiting). Image size: 238035014 bytes.' metadata: creationTimestamp: "2026-04-25T00:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:04:40Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99.18a970bbf16951eb namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: 3879bbb6-6dd5-4691-a9b7-01fd78b6c8a5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:04:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99 namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: fce959c1-497c-4e87-8e24-4b5ef6e69338 kind: Event lastTimestamp: "2026-04-25T00:04:40Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-25T00:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:04:40Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99.18a970bbf4983921 namespace: kserve-ci-e2e-test resourceVersion: "14517" uid: bc4fbf43-d191-4c91-bfcf-c6de49377b2a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:04:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99 namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: fce959c1-497c-4e87-8e24-4b5ef6e69338 kind: Event lastTimestamp: "2026-04-25T00:04:40Z" message: Started container agent metadata: creationTimestamp: "2026-04-25T00:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:04:40Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99.18a970bbf509d31d namespace: kserve-ci-e2e-test resourceVersion: "14518" uid: c90c8bfd-f799-442c-8c1f-99ac8e593092 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-25T00:04:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99 namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: fce959c1-497c-4e87-8e24-4b5ef6e69338 kind: Event lastTimestamp: "2026-04-25T00:06:13Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-25T00:04:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:06:13Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99.18a970bbfea1d73a namespace: kserve-ci-e2e-test resourceVersion: "15344" uid: cc287011-e1d4-4ae3-acf7-c26d835e3f5d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-25T00:04:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99 namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: fce959c1-497c-4e87-8e24-4b5ef6e69338 kind: Event lastTimestamp: "2026-04-25T00:06:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-25T00:04:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:06:13Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99.18a970bc3a6909f4 namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: 4bf3c291-0d40-4ce0-860c-848563b9e5b4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:05:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99 namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: fce959c1-497c-4e87-8e24-4b5ef6e69338 kind: Event lastTimestamp: "2026-04-25T00:05:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-25T00:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:05:58Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99.18a970ce21669cfd namespace: kserve-ci-e2e-test resourceVersion: "15123" uid: 812f3542-2fc2-44ca-8200-dcb223f8389b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:05:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99 namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: fce959c1-497c-4e87-8e24-4b5ef6e69338 kind: Event lastTimestamp: "2026-04-25T00:05:58Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-25T00:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:05:58Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99.18a970ce2166ee74 namespace: kserve-ci-e2e-test resourceVersion: "15126" uid: 6f570f25-7528-4c43-9b56-ba9d1892a67e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:05:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99 namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: fce959c1-497c-4e87-8e24-4b5ef6e69338 kind: Event lastTimestamp: "2026-04-25T00:05:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-25T00:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:05:58Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99.18a970ce21676413 namespace: kserve-ci-e2e-test resourceVersion: "15127" uid: c5183e82-b3b8-414c-bcff-0aba234b1356 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-25T00:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99 namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: fce959c1-497c-4e87-8e24-4b5ef6e69338 kind: Event lastTimestamp: "2026-04-25T00:06:17Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-25T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:06:17Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99.18a970cf169cffd2 namespace: kserve-ci-e2e-test resourceVersion: "15375" uid: 955e47f2-f0ca-4f75-a63b-1d651f67e3e2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:04:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9 namespace: kserve-ci-e2e-test resourceVersion: "14280" uid: 68aff513-9f4c-4421-bf31-3c18790e26a6 kind: Event lastTimestamp: "2026-04-25T00:04:13Z" message: 'Created pod: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9-44d99' metadata: creationTimestamp: "2026-04-25T00:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:04:13Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9.18a970b5ab5c0157 namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: e77386c8-eb38-4c62-91ae-77cea3ae3fa7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:04:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-8fb8f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: a170f7cb-e480-442e-bbee-e1e67b791ca6 kind: Event lastTimestamp: "2026-04-25T00:04:13Z" message: Scaled up replica set isvc-raw-sklearn-batcher-8fb8f-predictor-59f6ff8c9 from 0 to 1 metadata: creationTimestamp: "2026-04-25T00:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:04:13Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor.18a970b5a566f872 namespace: kserve-ci-e2e-test resourceVersion: "14282" uid: 1b69c52d-a0e5-4ba2-8e26-683fc529667d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-25T00:04:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-8fb8f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14291" uid: 0954d851-cd46-4e0d-aa46-04080d87c3a4 kind: Event lastTimestamp: "2026-04-25T00:05:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-25T00:04:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:05:13Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor.18a970b92b0686ec namespace: kserve-ci-e2e-test resourceVersion: "14758" uid: 334fda84-59ec-432a-89f0-09cd43917d18 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-25T00:04:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-8fb8f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14291" uid: 0954d851-cd46-4e0d-aa46-04080d87c3a4 kind: Event lastTimestamp: "2026-04-25T00:05:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-25T00:04:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:05:13Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor.18a970b92b915101 namespace: kserve-ci-e2e-test resourceVersion: "14759" uid: 8b0f81f0-ab80-4a74-a28b-266f1dd2d88e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-25T00:05:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-8fb8f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14424" uid: 0954d851-cd46-4e0d-aa46-04080d87c3a4 kind: Event lastTimestamp: "2026-04-25T00:05:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-25T00:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:05:43Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor.18a970c7275c3ac8 namespace: kserve-ci-e2e-test resourceVersion: "14975" uid: ae28c5ac-f2f2-43ec-85df-3ab9305de28c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-25T00:05:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-8fb8f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14424" uid: 0954d851-cd46-4e0d-aa46-04080d87c3a4 kind: Event lastTimestamp: "2026-04-25T00:05:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-25T00:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:05:43Z" name: isvc-raw-sklearn-batcher-8fb8f-predictor.18a970c727be8b35 namespace: kserve-ci-e2e-test resourceVersion: "14976" uid: d4aaf1c4-1ba0-4e3f-afb5-197059dc7f33 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-25T00:04:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-8fb8f namespace: kserve-ci-e2e-test resourceVersion: "14267" uid: 99185412-9714-47b1-b833-b3edebd9c4c2 kind: Event lastTimestamp: "2026-04-25T00:05:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-25T00:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:05:43Z" name: isvc-raw-sklearn-batcher-8fb8f.18a970b5ab927de7 namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: 9a014566-a274-4d9f-993c-e90781a7a5ba reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:04:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-8fb8f namespace: kserve-ci-e2e-test resourceVersion: "14267" uid: 99185412-9714-47b1-b833-b3edebd9c4c2 kind: Event lastTimestamp: "2026-04-25T00:04:13Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-8fb8f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-8fb8f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-25T00:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:04:13Z" name: isvc-raw-sklearn-batcher-8fb8f.18a970b5ac1d5635 namespace: kserve-ci-e2e-test resourceVersion: "14298" uid: 75d7ef88-5acd-4b0b-b4dd-625ec2d34507 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:04:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-8fb8f namespace: kserve-ci-e2e-test resourceVersion: "14267" uid: 99185412-9714-47b1-b833-b3edebd9c4c2 kind: Event lastTimestamp: "2026-04-25T00:04:13Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-8fb8f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-25T00:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:04:13Z" name: isvc-raw-sklearn-batcher-8fb8f.18a970b5ac1d9e40 namespace: kserve-ci-e2e-test resourceVersion: "14301" uid: 7338a53c-f3e8-4bc0-8d1f-2a9bf3122237 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:05:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-8fb8f namespace: kserve-ci-e2e-test resourceVersion: "14984" uid: 99185412-9714-47b1-b833-b3edebd9c4c2 kind: Event lastTimestamp: "2026-04-25T00:05:43Z" message: InferenceService [isvc-raw-sklearn-batcher-8fb8f] is Ready metadata: creationTimestamp: "2026-04-25T00:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:05:43Z" name: isvc-raw-sklearn-batcher-8fb8f.18a970cab0ee2652 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 9059ad73-f2ff-44cd-a089-2008a1f9349d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-25T00:15:05.081340Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-bbf281-predictor-7f76d5b559-s84pm namespace: kserve-ci-e2e-test resourceVersion: "19941" uid: 15cd8bda-e985-47ae-97fd-c320a2448ac4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-bbf281-predictor-7f76d5b559-s84pm to ip-10-0-132-64.ec2.internal metadata: creationTimestamp: "2026-04-25T00:15:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-25T00:15:05Z" name: isvc-secondary-bbf281-predictor-7f76d5b559-s84pm.18a9714d5e3558ac namespace: kserve-ci-e2e-test resourceVersion: "19946" uid: cf37205e-447d-42e5-81ef-c178c6d572fc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56c7745b65-967mt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-bbf281-predictor-7f76d5b559-s84pm namespace: kserve-ci-e2e-test resourceVersion: "19943" uid: 15cd8bda-e985-47ae-97fd-c320a2448ac4 kind: Event lastTimestamp: "2026-04-25T00:15:05Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-secondary-bbf281-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-25T00:15:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:15:05Z" name: isvc-secondary-bbf281-predictor-7f76d5b559-s84pm.18a9714d652253c0 namespace: kserve-ci-e2e-test resourceVersion: "19954" uid: 7e6a0278-d11d-41f5-9077-c53eb68f486c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:06Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-bbf281-predictor-7f76d5b559-s84pm namespace: kserve-ci-e2e-test resourceVersion: "19952" uid: 15cd8bda-e985-47ae-97fd-c320a2448ac4 kind: Event lastTimestamp: "2026-04-25T00:15:06Z" message: Add eth0 [10.134.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-25T00:15:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-25T00:15:06Z" name: isvc-secondary-bbf281-predictor-7f76d5b559-s84pm.18a9714d9b5ec914 namespace: kserve-ci-e2e-test resourceVersion: "19964" uid: 788cfbfb-5e96-45c1-96ea-a9bfec154627 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-25T00:15:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-bbf281-predictor-7f76d5b559-s84pm namespace: kserve-ci-e2e-test resourceVersion: "19943" uid: 15cd8bda-e985-47ae-97fd-c320a2448ac4 kind: Event lastTimestamp: "2026-04-25T00:15:10Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:72e9a53b4f6609f050c864ccf8bba3bc53b15bbbb0a09e5de74a3bdc7465dd84" already present on machine metadata: creationTimestamp: "2026-04-25T00:15:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:15:10Z" name: isvc-secondary-bbf281-predictor-7f76d5b559-s84pm.18a9714d9cd80be2 namespace: kserve-ci-e2e-test resourceVersion: "20003" uid: 533ef900-111b-4a93-9630-393b9c5201d2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-25T00:15:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-bbf281-predictor-7f76d5b559-s84pm namespace: kserve-ci-e2e-test resourceVersion: "19943" uid: 15cd8bda-e985-47ae-97fd-c320a2448ac4 kind: Event lastTimestamp: "2026-04-25T00:15:10Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-25T00:15:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:15:10Z" name: isvc-secondary-bbf281-predictor-7f76d5b559-s84pm.18a9714d9fcd171e namespace: kserve-ci-e2e-test resourceVersion: "20007" uid: 1f23e747-231c-4f25-9157-498d672d9156 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-25T00:15:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-bbf281-predictor-7f76d5b559-s84pm namespace: kserve-ci-e2e-test resourceVersion: "19943" uid: 15cd8bda-e985-47ae-97fd-c320a2448ac4 kind: Event lastTimestamp: "2026-04-25T00:15:10Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-25T00:15:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:15:10Z" name: isvc-secondary-bbf281-predictor-7f76d5b559-s84pm.18a9714da0472cef namespace: kserve-ci-e2e-test resourceVersion: "20008" uid: ff29e01f-446e-495e-bb00-0b518edc45b7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-bbf281-predictor-7f76d5b559-s84pm namespace: kserve-ci-e2e-test resourceVersion: "19943" uid: 15cd8bda-e985-47ae-97fd-c320a2448ac4 kind: Event lastTimestamp: "2026-04-25T00:15:14Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-bbf281-predictor-7f76d5b559-s84pm_kserve-ci-e2e-test(15cd8bda-e985-47ae-97fd-c320a2448ac4) metadata: creationTimestamp: "2026-04-25T00:15:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:15:14Z" name: isvc-secondary-bbf281-predictor-7f76d5b559-s84pm.18a9714f74e824df namespace: kserve-ci-e2e-test resourceVersion: "20039" uid: 2b493417-a01d-49b0-ad6d-e73005c1e9d1 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-bbf281-predictor-7f76d5b559 namespace: kserve-ci-e2e-test resourceVersion: "19932" uid: b242e1c2-9587-4a28-a37a-2cd68302f7c6 kind: Event lastTimestamp: "2026-04-25T00:15:05Z" message: 'Created pod: isvc-secondary-bbf281-predictor-7f76d5b559-s84pm' metadata: creationTimestamp: "2026-04-25T00:15:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:15:05Z" name: isvc-secondary-bbf281-predictor-7f76d5b559.18a9714d5dca988e namespace: kserve-ci-e2e-test resourceVersion: "19942" uid: c52a728f-3153-4f76-ac07-21675b4470e6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-bbf281-predictor namespace: kserve-ci-e2e-test resourceVersion: "19930" uid: b2d1de80-cc8b-4cba-acfb-818c288f44a0 kind: Event lastTimestamp: "2026-04-25T00:15:05Z" message: Scaled up replica set isvc-secondary-bbf281-predictor-7f76d5b559 from 0 to 1 metadata: creationTimestamp: "2026-04-25T00:15:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:15:05Z" name: isvc-secondary-bbf281-predictor.18a9714d59a570bb namespace: kserve-ci-e2e-test resourceVersion: "19934" uid: 4c1c5eca-909e-444c-8359-4275ef81d3c3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-bbf281-predictor namespace: kserve-ci-e2e-test resourceVersion: "19931" uid: 58228689-7870-4cd1-8fcf-dd8e6a5370fb kind: Event lastTimestamp: "2026-04-25T00:15:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-25T00:15:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:15:20Z" name: isvc-secondary-bbf281-predictor.18a97150d814468b namespace: kserve-ci-e2e-test resourceVersion: "20086" uid: 568e0e6b-cd71-4955-8615-e6ac1808471d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-bbf281-predictor namespace: kserve-ci-e2e-test resourceVersion: "19931" uid: 58228689-7870-4cd1-8fcf-dd8e6a5370fb kind: Event lastTimestamp: "2026-04-25T00:15:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-25T00:15:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:15:20Z" name: isvc-secondary-bbf281-predictor.18a97150d8814c5f namespace: kserve-ci-e2e-test resourceVersion: "20088" uid: fe21e87b-b6af-44f2-ba36-b9baadfe3d61 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 14 eventTime: null firstTimestamp: "2026-04-25T00:15:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-bbf281 namespace: kserve-ci-e2e-test resourceVersion: "19921" uid: 9b32b437-562a-4f92-9c29-2f1935715517 kind: Event lastTimestamp: "2026-04-25T00:15:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-25T00:15:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:15:16Z" name: isvc-secondary-bbf281.18a9714d59896d73 namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 1a400d1e-4862-4fb6-9410-5648d5d7c5ef reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-25T00:15:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-bbf281 namespace: kserve-ci-e2e-test resourceVersion: "19921" uid: 9b32b437-562a-4f92-9c29-2f1935715517 kind: Event lastTimestamp: "2026-04-25T00:15:05Z" message: 'Failed to update status for InferenceService "isvc-secondary-bbf281": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-bbf281": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-25T00:15:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:15:05Z" name: isvc-secondary-bbf281.18a9714d5a014c2b namespace: kserve-ci-e2e-test resourceVersion: "19947" uid: 4be9da29-8fee-4d4a-ac42-3d9a81cff339 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-25T00:15:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-bbf281 namespace: kserve-ci-e2e-test resourceVersion: "19921" uid: 9b32b437-562a-4f92-9c29-2f1935715517 kind: Event lastTimestamp: "2026-04-25T00:15:05Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-bbf281": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-25T00:15:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:15:05Z" name: isvc-secondary-bbf281.18a9714d5a01ba5d namespace: kserve-ci-e2e-test resourceVersion: "19950" uid: e2d7db14-3d4f-4964-996b-b4a3ae10d912 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-25T00:05:58.686924Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv namespace: kserve-ci-e2e-test resourceVersion: "15138" uid: 051a1f66-e6a7-47a9-8eab-ec1836b062fa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv to ip-10-0-132-64.ec2.internal metadata: creationTimestamp: "2026-04-25T00:05:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-25T00:05:58Z" name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv.18a970ce26904736 namespace: kserve-ci-e2e-test resourceVersion: "15150" uid: 31e2a52e-5a6c-4a16-8bc7-0e30b0842f8d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56c7745b65-967mt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:05:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: 051a1f66-e6a7-47a9-8eab-ec1836b062fa kind: Event lastTimestamp: "2026-04-25T00:05:59Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-25T00:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-25T00:05:59Z" name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv.18a970ce4044ab86 namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: 570d81a9-8759-4824-9bef-2e8772b04053 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:05:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: 051a1f66-e6a7-47a9-8eab-ec1836b062fa kind: Event lastTimestamp: "2026-04-25T00:05:59Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:72e9a53b4f6609f050c864ccf8bba3bc53b15bbbb0a09e5de74a3bdc7465dd84" already present on machine metadata: creationTimestamp: "2026-04-25T00:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:05:59Z" name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv.18a970ce41b09601 namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: aab51e34-dd32-4d07-81d0-aa89eab8571f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:05:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: 051a1f66-e6a7-47a9-8eab-ec1836b062fa kind: Event lastTimestamp: "2026-04-25T00:05:59Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-25T00:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:05:59Z" name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv.18a970ce44e77cdf namespace: kserve-ci-e2e-test resourceVersion: "15190" uid: 33f83094-a907-43f6-9770-0bc44c69157a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:05:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: 051a1f66-e6a7-47a9-8eab-ec1836b062fa kind: Event lastTimestamp: "2026-04-25T00:05:59Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-25T00:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:05:59Z" name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv.18a970ce455b4ab1 namespace: kserve-ci-e2e-test resourceVersion: "15192" uid: da5629db-d1b0-415f-98f6-087a1464b358 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:06:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: 051a1f66-e6a7-47a9-8eab-ec1836b062fa kind: Event lastTimestamp: "2026-04-25T00:06:04Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1435" already present on machine metadata: creationTimestamp: "2026-04-25T00:06:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:06:04Z" name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv.18a970cf6390bc37 namespace: kserve-ci-e2e-test resourceVersion: "15250" uid: 2661a54f-ecfa-4296-aeeb-248ce8c04edd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:06:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: 051a1f66-e6a7-47a9-8eab-ec1836b062fa kind: Event lastTimestamp: "2026-04-25T00:06:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-25T00:06:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:06:04Z" name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv.18a970cf66d8a92a namespace: kserve-ci-e2e-test resourceVersion: "15254" uid: 9abce433-8523-45e9-9b2d-a70be33e818e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:06:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: 051a1f66-e6a7-47a9-8eab-ec1836b062fa kind: Event lastTimestamp: "2026-04-25T00:06:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-25T00:06:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:06:04Z" name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv.18a970cf676adeb9 namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: c89d94b8-6a50-46f7-b31e-d7bb56fbeed5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:06:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: 051a1f66-e6a7-47a9-8eab-ec1836b062fa kind: Event lastTimestamp: "2026-04-25T00:06:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-25T00:06:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:06:04Z" name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv.18a970cf6778439b namespace: kserve-ci-e2e-test resourceVersion: "15256" uid: 78e86d22-e2ba-4201-b6b1-1ccd3d18fa66 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:06:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: 051a1f66-e6a7-47a9-8eab-ec1836b062fa kind: Event lastTimestamp: "2026-04-25T00:06:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-25T00:06:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:06:04Z" name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv.18a970cf6aa75125 namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: 3102af93-4adf-4164-a441-510c9c4874b4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:06:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: 051a1f66-e6a7-47a9-8eab-ec1836b062fa kind: Event lastTimestamp: "2026-04-25T00:06:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-25T00:06:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:06:04Z" name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv.18a970cf6b1b2505 namespace: kserve-ci-e2e-test resourceVersion: "15259" uid: f765dca9-a340-49af-8a0d-1cea8bdf05d0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-25T00:06:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: 051a1f66-e6a7-47a9-8eab-ec1836b062fa kind: Event lastTimestamp: "2026-04-25T00:07:51Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-25T00:06:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:51Z" name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv.18a970cf9f8e8887 namespace: kserve-ci-e2e-test resourceVersion: "16275" uid: 330e1b2c-73fd-4ad9-9fae-4227684150ee reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: 051a1f66-e6a7-47a9-8eab-ec1836b062fa kind: Event lastTimestamp: "2026-04-25T00:07:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-25T00:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:48Z" name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv.18a970e7d2ccf0bb namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: 4d12538f-4a29-46a0-9892-7e96fd82136f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: 051a1f66-e6a7-47a9-8eab-ec1836b062fa kind: Event lastTimestamp: "2026-04-25T00:07:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-25T00:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:48Z" name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv.18a970e7d2cd69e9 namespace: kserve-ci-e2e-test resourceVersion: "16172" uid: d07a0349-1fdf-4593-a342-0c3d187b8a77 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: 051a1f66-e6a7-47a9-8eab-ec1836b062fa kind: Event lastTimestamp: "2026-04-25T00:07:51Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp 10.134.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-25T00:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:51Z" name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv.18a970e84dcab971 namespace: kserve-ci-e2e-test resourceVersion: "16274" uid: 5e36ca08-b2ea-4f7f-ab56-a7a4122f90e3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:05:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: 3c04356f-46e8-4523-9806-c3c10f3b3001 kind: Event lastTimestamp: "2026-04-25T00:05:58Z" message: 'Created pod: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c-ffgpv' metadata: creationTimestamp: "2026-04-25T00:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:05:58Z" name: isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c.18a970ce2621bea5 namespace: kserve-ci-e2e-test resourceVersion: "15141" uid: 6ac4450d-57e6-4277-88a7-87e049e15712 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:05:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-3b499-predictor namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 0e84bfbf-8b92-4ea1-bc82-60c29921233c kind: Event lastTimestamp: "2026-04-25T00:05:58Z" message: Scaled up replica set isvc-sklearn-graph-raw-3b499-predictor-756fc45b6c from 0 to 1 metadata: creationTimestamp: "2026-04-25T00:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:05:58Z" name: isvc-sklearn-graph-raw-3b499-predictor.18a970ce217353e7 namespace: kserve-ci-e2e-test resourceVersion: "15124" uid: 2f95e3aa-d2f4-4c27-8b5e-4ff495576c16 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:06:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-3b499-predictor namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: 73d93441-c0b1-4416-a1a7-798eaf1b6116 kind: Event lastTimestamp: "2026-04-25T00:06:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-25T00:06:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:06:13Z" name: isvc-sklearn-graph-raw-3b499-predictor.18a970d19ff01740 namespace: kserve-ci-e2e-test resourceVersion: "15338" uid: d8c42215-837d-474d-bed8-c03f6b5bc820 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:06:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-3b499-predictor namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: 73d93441-c0b1-4416-a1a7-798eaf1b6116 kind: Event lastTimestamp: "2026-04-25T00:06:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-25T00:06:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:06:13Z" name: isvc-sklearn-graph-raw-3b499-predictor.18a970d1a05991ec namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: a60724c4-a1bc-4e0f-bcdd-5970026cca03 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-25T00:06:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-3b499-predictor namespace: kserve-ci-e2e-test resourceVersion: "15339" uid: 73d93441-c0b1-4416-a1a7-798eaf1b6116 kind: Event lastTimestamp: "2026-04-25T00:07:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-25T00:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:07:43Z" name: isvc-sklearn-graph-raw-3b499-predictor.18a970d521ddd444 namespace: kserve-ci-e2e-test resourceVersion: "16073" uid: 5722eea1-c57e-4817-9ea7-abce2c88c555 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-25T00:06:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-3b499-predictor namespace: kserve-ci-e2e-test resourceVersion: "15339" uid: 73d93441-c0b1-4416-a1a7-798eaf1b6116 kind: Event lastTimestamp: "2026-04-25T00:07:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-25T00:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:07:43Z" name: isvc-sklearn-graph-raw-3b499-predictor.18a970d522a4c0ee namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: a48a1bf5-907b-42ba-b00e-a270d5c6330b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-25T00:05:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-3b499 namespace: kserve-ci-e2e-test resourceVersion: "15092" uid: 727859c9-0806-463d-ab98-f00a6f666953 kind: Event lastTimestamp: "2026-04-25T00:07:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-25T00:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:07:11Z" name: isvc-sklearn-graph-raw-3b499.18a970ce215473a8 namespace: kserve-ci-e2e-test resourceVersion: "15792" uid: 86a24ada-16ea-452d-80ec-baababbc5abf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:05:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-3b499 namespace: kserve-ci-e2e-test resourceVersion: "15092" uid: 727859c9-0806-463d-ab98-f00a6f666953 kind: Event lastTimestamp: "2026-04-25T00:05:58Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-3b499": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-3b499": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-25T00:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:05:58Z" name: isvc-sklearn-graph-raw-3b499.18a970ce21f86c52 namespace: kserve-ci-e2e-test resourceVersion: "15128" uid: 2847174d-c29b-4ad6-9363-eada704b1676 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-3b499 namespace: kserve-ci-e2e-test resourceVersion: "15790" uid: 727859c9-0806-463d-ab98-f00a6f666953 kind: Event lastTimestamp: "2026-04-25T00:07:11Z" message: InferenceService [isvc-sklearn-graph-raw-3b499] is Ready metadata: creationTimestamp: "2026-04-25T00:07:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:07:11Z" name: isvc-sklearn-graph-raw-3b499.18a970df025e8199 namespace: kserve-ci-e2e-test resourceVersion: "15791" uid: da32152e-5248-41c2-9d96-83b45a4bdccc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-25T00:07:48.994099Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: 94574d49-dd0d-4611-b527-a47d1fe2edca kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt to ip-10-0-132-64.ec2.internal metadata: creationTimestamp: "2026-04-25T00:07:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-25T00:07:48Z" name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt.18a970e7d5622bc5 namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: c06f5872-9525-4858-b8c8-e9f5077d4c2f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56c7745b65-967mt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:49Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt namespace: kserve-ci-e2e-test resourceVersion: "16178" uid: 94574d49-dd0d-4611-b527-a47d1fe2edca kind: Event lastTimestamp: "2026-04-25T00:07:49Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-raw-hpa-3f0c4-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-25T00:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:49Z" name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt.18a970e7dd04e0a8 namespace: kserve-ci-e2e-test resourceVersion: "16228" uid: 59e75d25-0bdc-46eb-92ff-5dc9488038b7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:50Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt namespace: kserve-ci-e2e-test resourceVersion: "16191" uid: 94574d49-dd0d-4611-b527-a47d1fe2edca kind: Event lastTimestamp: "2026-04-25T00:07:50Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-25T00:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-25T00:07:50Z" name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt.18a970e81306ff74 namespace: kserve-ci-e2e-test resourceVersion: "16256" uid: 305e2224-1437-49fe-8d8d-430b563fb9ca reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt namespace: kserve-ci-e2e-test resourceVersion: "16178" uid: 94574d49-dd0d-4611-b527-a47d1fe2edca kind: Event lastTimestamp: "2026-04-25T00:07:50Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:72e9a53b4f6609f050c864ccf8bba3bc53b15bbbb0a09e5de74a3bdc7465dd84" already present on machine metadata: creationTimestamp: "2026-04-25T00:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:50Z" name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt.18a970e814548236 namespace: kserve-ci-e2e-test resourceVersion: "16259" uid: bb406c3a-dfa7-4572-8a88-141d4f977b97 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt namespace: kserve-ci-e2e-test resourceVersion: "16178" uid: 94574d49-dd0d-4611-b527-a47d1fe2edca kind: Event lastTimestamp: "2026-04-25T00:07:50Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-25T00:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:50Z" name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt.18a970e8173788cb namespace: kserve-ci-e2e-test resourceVersion: "16260" uid: 9934f961-695f-42cc-9f03-e63fc1d14252 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt namespace: kserve-ci-e2e-test resourceVersion: "16178" uid: 94574d49-dd0d-4611-b527-a47d1fe2edca kind: Event lastTimestamp: "2026-04-25T00:07:50Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-25T00:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:50Z" name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt.18a970e817aac531 namespace: kserve-ci-e2e-test resourceVersion: "16261" uid: bff3efa6-f74e-4c17-8664-01660f15bd07 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt namespace: kserve-ci-e2e-test resourceVersion: "16178" uid: 94574d49-dd0d-4611-b527-a47d1fe2edca kind: Event lastTimestamp: "2026-04-25T00:07:54Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1435" already present on machine metadata: creationTimestamp: "2026-04-25T00:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:54Z" name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt.18a970e918b45174 namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 00cd1490-1086-430f-bab1-7f3977d26659 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt namespace: kserve-ci-e2e-test resourceVersion: "16178" uid: 94574d49-dd0d-4611-b527-a47d1fe2edca kind: Event lastTimestamp: "2026-04-25T00:07:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-25T00:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:54Z" name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt.18a970e91cd8113e namespace: kserve-ci-e2e-test resourceVersion: "16324" uid: bee8665b-0272-4f1f-80bc-228da6a6dc78 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt namespace: kserve-ci-e2e-test resourceVersion: "16178" uid: 94574d49-dd0d-4611-b527-a47d1fe2edca kind: Event lastTimestamp: "2026-04-25T00:07:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-25T00:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:54Z" name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt.18a970e91d513e1b namespace: kserve-ci-e2e-test resourceVersion: "16325" uid: f05fa1b4-5923-400e-b129-84ce21806edc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt namespace: kserve-ci-e2e-test resourceVersion: "16178" uid: 94574d49-dd0d-4611-b527-a47d1fe2edca kind: Event lastTimestamp: "2026-04-25T00:07:54Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-25T00:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:54Z" name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt.18a970e91d5e6f06 namespace: kserve-ci-e2e-test resourceVersion: "16326" uid: 21dd5ef0-1a2f-49d2-af67-94cc85ed520e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt namespace: kserve-ci-e2e-test resourceVersion: "16178" uid: 94574d49-dd0d-4611-b527-a47d1fe2edca kind: Event lastTimestamp: "2026-04-25T00:07:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-25T00:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:54Z" name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt.18a970e92062fc65 namespace: kserve-ci-e2e-test resourceVersion: "16328" uid: d64ef87f-2179-4414-ae4d-7722a552c1dd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt namespace: kserve-ci-e2e-test resourceVersion: "16178" uid: 94574d49-dd0d-4611-b527-a47d1fe2edca kind: Event lastTimestamp: "2026-04-25T00:07:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-25T00:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:54Z" name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt.18a970e920d62fa9 namespace: kserve-ci-e2e-test resourceVersion: "16329" uid: 92f4fd70-fc33-4b19-bd17-4a26e0f13e1a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-25T00:07:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt namespace: kserve-ci-e2e-test resourceVersion: "16178" uid: 94574d49-dd0d-4611-b527-a47d1fe2edca kind: Event lastTimestamp: "2026-04-25T00:09:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-25T00:07:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:32Z" name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt.18a970e990735f40 namespace: kserve-ci-e2e-test resourceVersion: "17229" uid: 994fde08-1864-4a6d-b60f-ca235490107e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt namespace: kserve-ci-e2e-test resourceVersion: "16178" uid: 94574d49-dd0d-4611-b527-a47d1fe2edca kind: Event lastTimestamp: "2026-04-25T00:09:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-25T00:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:29Z" name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt.18a970ff2d07b348 namespace: kserve-ci-e2e-test resourceVersion: "17151" uid: 8fe6eb24-72ae-4db8-9aab-83dd3828a942 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt namespace: kserve-ci-e2e-test resourceVersion: "16178" uid: 94574d49-dd0d-4611-b527-a47d1fe2edca kind: Event lastTimestamp: "2026-04-25T00:09:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-25T00:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:29Z" name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt.18a970ff2d0829d9 namespace: kserve-ci-e2e-test resourceVersion: "17153" uid: 9edde72c-d244-4af7-b92f-f1a92df327d5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt namespace: kserve-ci-e2e-test resourceVersion: "16178" uid: 94574d49-dd0d-4611-b527-a47d1fe2edca kind: Event lastTimestamp: "2026-04-25T00:09:32Z" message: 'Readiness probe failed: Get "https://10.134.0.40:8643/healthz": dial tcp 10.134.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-25T00:09:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:32Z" name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt.18a970ffeaaddfbb namespace: kserve-ci-e2e-test resourceVersion: "17228" uid: f95f0528-5fed-436b-aa78-2c411fbe59bb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9 namespace: kserve-ci-e2e-test resourceVersion: "16152" uid: f61830a5-1fc7-42e1-aa8b-c7655d68ec36 kind: Event lastTimestamp: "2026-04-25T00:07:48Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9-trsgt' metadata: creationTimestamp: "2026-04-25T00:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:07:48Z" name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9.18a970e7d4dc0f9c namespace: kserve-ci-e2e-test resourceVersion: "16177" uid: 5957be01-8794-4719-81c8-e75dc0fc95bb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor namespace: kserve-ci-e2e-test resourceVersion: "16144" uid: 73a90566-9991-4982-8205-ff2eb677df69 kind: Event lastTimestamp: "2026-04-25T00:07:48Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-3f0c4-predictor-759c5664f9 from 0 to 1 metadata: creationTimestamp: "2026-04-25T00:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:07:48Z" name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor.18a970e7d0e8c971 namespace: kserve-ci-e2e-test resourceVersion: "16156" uid: a7c4d0e8-87e0-4608-b47e-395a31f7655a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-25T00:08:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor namespace: kserve-ci-e2e-test resourceVersion: "16150" uid: 90c8acb2-0459-4a12-a5fb-06b37d1a83ed kind: Event lastTimestamp: "2026-04-25T00:08:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-25T00:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:08:18Z" name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor.18a970eb4f6493fb namespace: kserve-ci-e2e-test resourceVersion: "16521" uid: 6490d201-0ac1-4e31-8767-3097f5eb575b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-25T00:08:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor namespace: kserve-ci-e2e-test resourceVersion: "16150" uid: 90c8acb2-0459-4a12-a5fb-06b37d1a83ed kind: Event lastTimestamp: "2026-04-25T00:08:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-25T00:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:08:18Z" name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor.18a970eb4fd2b060 namespace: kserve-ci-e2e-test resourceVersion: "16522" uid: b5dfd429-26b2-4d98-b68d-3292ee362de6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-25T00:08:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor namespace: kserve-ci-e2e-test resourceVersion: "16411" uid: 90c8acb2-0459-4a12-a5fb-06b37d1a83ed kind: Event lastTimestamp: "2026-04-25T00:09:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-25T00:08:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:09:19Z" name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor.18a970f24d981062 namespace: kserve-ci-e2e-test resourceVersion: "17000" uid: 5acf8fd5-8f85-48c6-8b5d-4a0fd5e542c7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-25T00:08:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor namespace: kserve-ci-e2e-test resourceVersion: "16411" uid: 90c8acb2-0459-4a12-a5fb-06b37d1a83ed kind: Event lastTimestamp: "2026-04-25T00:09:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-25T00:08:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:09:19Z" name: isvc-sklearn-graph-raw-hpa-3f0c4-predictor.18a970f24dfbaab6 namespace: kserve-ci-e2e-test resourceVersion: "17001" uid: a85b0366-1c0f-4c8b-a263-745dd93648a1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-25T00:07:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-3f0c4 namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: 608f6126-023a-49d9-a736-41551a9fabe4 kind: Event lastTimestamp: "2026-04-25T00:09:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-25T00:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:09:02Z" name: isvc-sklearn-graph-raw-hpa-3f0c4.18a970e7d0c65605 namespace: kserve-ci-e2e-test resourceVersion: "16869" uid: 0acf6de5-cf84-4180-8b4a-15f5c5cacdf8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-3f0c4 namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: 608f6126-023a-49d9-a736-41551a9fabe4 kind: Event lastTimestamp: "2026-04-25T00:07:48Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-3f0c4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-3f0c4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-25T00:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:07:48Z" name: isvc-sklearn-graph-raw-hpa-3f0c4.18a970e7d166331c namespace: kserve-ci-e2e-test resourceVersion: "16161" uid: bbfaefa0-f05f-40be-a6f1-cfccca797b58 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-3f0c4 namespace: kserve-ci-e2e-test resourceVersion: "16867" uid: 608f6126-023a-49d9-a736-41551a9fabe4 kind: Event lastTimestamp: "2026-04-25T00:09:02Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-3f0c4] is Ready metadata: creationTimestamp: "2026-04-25T00:09:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:09:02Z" name: isvc-sklearn-graph-raw-hpa-3f0c4.18a970f8f3dbbd80 namespace: kserve-ci-e2e-test resourceVersion: "16868" uid: 01bbe68a-ff00-4b0b-9166-0eb241217f99 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-25T00:11:04.601270Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d namespace: kserve-ci-e2e-test resourceVersion: "18038" uid: 2701b787-c103-4814-829e-91bf7d8efe16 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d to ip-10-0-132-64.ec2.internal metadata: creationTimestamp: "2026-04-25T00:11:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-25T00:11:04Z" name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d.18a97115607aacc2 namespace: kserve-ci-e2e-test resourceVersion: "18044" uid: 96e31183-fe72-4e17-830f-61c01521f2a7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56c7745b65-967mt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:11:04Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d namespace: kserve-ci-e2e-test resourceVersion: "18042" uid: 2701b787-c103-4814-829e-91bf7d8efe16 kind: Event lastTimestamp: "2026-04-25T00:11:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-scale-raw-e81d6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-25T00:11:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:11:04Z" name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d.18a971156b8bb0a4 namespace: kserve-ci-e2e-test resourceVersion: "18068" uid: 4a871245-8514-4da6-94fe-1a221913a2bb reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:11:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d namespace: kserve-ci-e2e-test resourceVersion: "18053" uid: 2701b787-c103-4814-829e-91bf7d8efe16 kind: Event lastTimestamp: "2026-04-25T00:11:05Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-25T00:11:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-25T00:11:05Z" name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d.18a971159e08616d namespace: kserve-ci-e2e-test resourceVersion: "18080" uid: 51cf5154-4d3a-4af4-b8b2-95ca21b68efe reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:11:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d namespace: kserve-ci-e2e-test resourceVersion: "18042" uid: 2701b787-c103-4814-829e-91bf7d8efe16 kind: Event lastTimestamp: "2026-04-25T00:11:05Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:72e9a53b4f6609f050c864ccf8bba3bc53b15bbbb0a09e5de74a3bdc7465dd84" already present on machine metadata: creationTimestamp: "2026-04-25T00:11:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:11:05Z" name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d.18a971159f679523 namespace: kserve-ci-e2e-test resourceVersion: "18083" uid: 4734c0ed-0aa1-4127-875f-015c92cfcb3c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:11:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d namespace: kserve-ci-e2e-test resourceVersion: "18042" uid: 2701b787-c103-4814-829e-91bf7d8efe16 kind: Event lastTimestamp: "2026-04-25T00:11:05Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-25T00:11:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:11:05Z" name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d.18a97115a2f8daa3 namespace: kserve-ci-e2e-test resourceVersion: "18088" uid: f7291e0b-7898-4cad-b988-c3bf723aa504 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:11:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d namespace: kserve-ci-e2e-test resourceVersion: "18042" uid: 2701b787-c103-4814-829e-91bf7d8efe16 kind: Event lastTimestamp: "2026-04-25T00:11:05Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-25T00:11:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:11:05Z" name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d.18a97115a3719f03 namespace: kserve-ci-e2e-test resourceVersion: "18089" uid: 343b3446-ff84-4b1f-b801-8293902ff32e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:11:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d namespace: kserve-ci-e2e-test resourceVersion: "18042" uid: 2701b787-c103-4814-829e-91bf7d8efe16 kind: Event lastTimestamp: "2026-04-25T00:11:10Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1435" already present on machine metadata: creationTimestamp: "2026-04-25T00:11:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:11:10Z" name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d.18a97116ab35bc4a namespace: kserve-ci-e2e-test resourceVersion: "18127" uid: 24be5d83-1027-43cd-ab8f-38004cf7d23e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:11:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d namespace: kserve-ci-e2e-test resourceVersion: "18042" uid: 2701b787-c103-4814-829e-91bf7d8efe16 kind: Event lastTimestamp: "2026-04-25T00:11:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-25T00:11:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:11:10Z" name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d.18a97116ae5f27c8 namespace: kserve-ci-e2e-test resourceVersion: "18131" uid: 9b36e911-3598-428d-a732-7a0986ea1105 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:11:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d namespace: kserve-ci-e2e-test resourceVersion: "18042" uid: 2701b787-c103-4814-829e-91bf7d8efe16 kind: Event lastTimestamp: "2026-04-25T00:11:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-25T00:11:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:11:10Z" name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d.18a97116aed953b3 namespace: kserve-ci-e2e-test resourceVersion: "18132" uid: 144017b2-591a-408c-8359-ea2d738957a1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:11:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d namespace: kserve-ci-e2e-test resourceVersion: "18042" uid: 2701b787-c103-4814-829e-91bf7d8efe16 kind: Event lastTimestamp: "2026-04-25T00:11:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-25T00:11:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:11:10Z" name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d.18a97116aee58b83 namespace: kserve-ci-e2e-test resourceVersion: "18133" uid: b3fa2085-0a9d-4b34-81f4-13af901708b7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:11:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d namespace: kserve-ci-e2e-test resourceVersion: "18042" uid: 2701b787-c103-4814-829e-91bf7d8efe16 kind: Event lastTimestamp: "2026-04-25T00:11:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-25T00:11:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:11:10Z" name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d.18a97116b28c8ea2 namespace: kserve-ci-e2e-test resourceVersion: "18136" uid: 30abfed2-5dc6-4459-83d3-483dd930e116 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:11:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d namespace: kserve-ci-e2e-test resourceVersion: "18042" uid: 2701b787-c103-4814-829e-91bf7d8efe16 kind: Event lastTimestamp: "2026-04-25T00:11:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-25T00:11:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:11:10Z" name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d.18a97116b31b5b4e namespace: kserve-ci-e2e-test resourceVersion: "18137" uid: cc9c9919-7627-4ba5-b650-d6f36364c94d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 17 eventTime: null firstTimestamp: "2026-04-25T00:11:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d namespace: kserve-ci-e2e-test resourceVersion: "18042" uid: 2701b787-c103-4814-829e-91bf7d8efe16 kind: Event lastTimestamp: "2026-04-25T00:13:38Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-25T00:11:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:13:38Z" name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d.18a9711723244256 namespace: kserve-ci-e2e-test resourceVersion: "19278" uid: 718023a2-6d1a-400a-a701-7e95237fb60f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:13:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d namespace: kserve-ci-e2e-test resourceVersion: "18042" uid: 2701b787-c103-4814-829e-91bf7d8efe16 kind: Event lastTimestamp: "2026-04-25T00:13:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-25T00:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:13:34Z" name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d.18a971385840acd8 namespace: kserve-ci-e2e-test resourceVersion: "19207" uid: fcd5b7b8-9e84-49c9-b29d-8e9015ba732a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:13:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d namespace: kserve-ci-e2e-test resourceVersion: "18042" uid: 2701b787-c103-4814-829e-91bf7d8efe16 kind: Event lastTimestamp: "2026-04-25T00:13:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-25T00:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:13:34Z" name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d.18a97138584182fd namespace: kserve-ci-e2e-test resourceVersion: "19209" uid: a8b61ab9-be34-4fc9-a561-c03932fc61a2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-25T00:13:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d namespace: kserve-ci-e2e-test resourceVersion: "18042" uid: 2701b787-c103-4814-829e-91bf7d8efe16 kind: Event lastTimestamp: "2026-04-25T00:13:43Z" message: 'Readiness probe failed: Get "https://10.134.0.43:8643/healthz": dial tcp 10.134.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-25T00:13:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:13:43Z" name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d.18a971392196e198 namespace: kserve-ci-e2e-test resourceVersion: "19325" uid: c7a11e81-8729-4864-8c44-edbc69b2a849 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:11:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65 namespace: kserve-ci-e2e-test resourceVersion: "18024" uid: 49c68ac1-4abb-4c60-8dd5-00958e6e6b67 kind: Event lastTimestamp: "2026-04-25T00:11:04Z" message: 'Created pod: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65-zmk7d' metadata: creationTimestamp: "2026-04-25T00:11:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:11:04Z" name: isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65.18a971155ff198e7 namespace: kserve-ci-e2e-test resourceVersion: "18041" uid: 818365d2-9ead-491a-8418-094f5f5e5f26 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:11:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-e81d6-predictor namespace: kserve-ci-e2e-test resourceVersion: "18019" uid: 1bc482d7-bbae-4646-8290-b768375f523e kind: Event lastTimestamp: "2026-04-25T00:11:04Z" message: Scaled up replica set isvc-sklearn-scale-raw-e81d6-predictor-6c8678b65 from 0 to 1 metadata: creationTimestamp: "2026-04-25T00:11:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:11:04Z" name: isvc-sklearn-scale-raw-e81d6-predictor.18a971155cf5b7a4 namespace: kserve-ci-e2e-test resourceVersion: "18026" uid: 5a61d15d-8cf8-49b9-8400-51c32602aac6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-25T00:11:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-e81d6-predictor namespace: kserve-ci-e2e-test resourceVersion: "18023" uid: 0ab2bd27-049d-4135-9a3c-204b00324e82 kind: Event lastTimestamp: "2026-04-25T00:11:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-25T00:11:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:11:49Z" name: isvc-sklearn-scale-raw-e81d6-predictor.18a97118dbfdb5cc namespace: kserve-ci-e2e-test resourceVersion: "18429" uid: cdced4cb-cb01-4cc8-955d-2f583ece84cd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-25T00:11:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-e81d6-predictor namespace: kserve-ci-e2e-test resourceVersion: "18023" uid: 0ab2bd27-049d-4135-9a3c-204b00324e82 kind: Event lastTimestamp: "2026-04-25T00:11:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-25T00:11:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:11:49Z" name: isvc-sklearn-scale-raw-e81d6-predictor.18a97118dc7161fb namespace: kserve-ci-e2e-test resourceVersion: "18430" uid: aa08acaf-2311-4e40-bc05-bd14b7239067 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-25T00:11:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-e81d6-predictor namespace: kserve-ci-e2e-test resourceVersion: "18209" uid: 0ab2bd27-049d-4135-9a3c-204b00324e82 kind: Event lastTimestamp: "2026-04-25T00:13:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-25T00:11:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:13:34Z" name: isvc-sklearn-scale-raw-e81d6-predictor.18a9711c5b2ed81b namespace: kserve-ci-e2e-test resourceVersion: "19180" uid: bf75e3b0-0c63-4661-9350-070ee1d289fe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-25T00:11:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-e81d6-predictor namespace: kserve-ci-e2e-test resourceVersion: "18209" uid: 0ab2bd27-049d-4135-9a3c-204b00324e82 kind: Event lastTimestamp: "2026-04-25T00:13:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-25T00:11:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:13:34Z" name: isvc-sklearn-scale-raw-e81d6-predictor.18a9711c5b941085 namespace: kserve-ci-e2e-test resourceVersion: "19181" uid: 998d945b-ab96-4bfa-bcc7-c124318a950f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-25T00:11:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-e81d6 namespace: kserve-ci-e2e-test resourceVersion: "17999" uid: da844e0f-fd5a-4486-9731-aef6ad2a258f kind: Event lastTimestamp: "2026-04-25T00:13:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-25T00:11:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:13:28Z" name: isvc-sklearn-scale-raw-e81d6.18a971155cfd9fe7 namespace: kserve-ci-e2e-test resourceVersion: "19131" uid: 00db5e2d-1a3b-47c0-90c6-97ef7d08ca07 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:11:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-e81d6 namespace: kserve-ci-e2e-test resourceVersion: "17999" uid: da844e0f-fd5a-4486-9731-aef6ad2a258f kind: Event lastTimestamp: "2026-04-25T00:11:04Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-e81d6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-e81d6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-25T00:11:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:11:04Z" name: isvc-sklearn-scale-raw-e81d6.18a971155d992152 namespace: kserve-ci-e2e-test resourceVersion: "18033" uid: 5703f801-a275-47ab-9158-130f36790c32 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:13:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-e81d6 namespace: kserve-ci-e2e-test resourceVersion: "19129" uid: da844e0f-fd5a-4486-9731-aef6ad2a258f kind: Event lastTimestamp: "2026-04-25T00:13:28Z" message: InferenceService [isvc-sklearn-scale-raw-e81d6] is Ready metadata: creationTimestamp: "2026-04-25T00:13:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:13:28Z" name: isvc-sklearn-scale-raw-e81d6.18a97136e3f610e6 namespace: kserve-ci-e2e-test resourceVersion: "19130" uid: 41805150-166c-4ee8-80e8-f8fdb29bbd69 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-25T00:05:58.760709Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7 namespace: kserve-ci-e2e-test resourceVersion: "15164" uid: 505acb2b-854c-4ba7-a80f-68509f5fee55 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7 to ip-10-0-139-62.ec2.internal metadata: creationTimestamp: "2026-04-25T00:05:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-25T00:05:58Z" name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7.18a970ce2af61eb5 namespace: kserve-ci-e2e-test resourceVersion: "15173" uid: 677389c9-e84a-47f3-98e4-c02df2b33bf8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56c7745b65-967mt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:05:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7 namespace: kserve-ci-e2e-test resourceVersion: "15178" uid: 505acb2b-854c-4ba7-a80f-68509f5fee55 kind: Event lastTimestamp: "2026-04-25T00:05:59Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-25T00:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-25T00:05:59Z" name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7.18a970ce447f3211 namespace: kserve-ci-e2e-test resourceVersion: "15189" uid: 5087f508-f2e9-4421-8e90-c4e6a65a1a8e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:05:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7 namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: 505acb2b-854c-4ba7-a80f-68509f5fee55 kind: Event lastTimestamp: "2026-04-25T00:05:59Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:72e9a53b4f6609f050c864ccf8bba3bc53b15bbbb0a09e5de74a3bdc7465dd84" metadata: creationTimestamp: "2026-04-25T00:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:05:59Z" name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7.18a970ce45f0301c namespace: kserve-ci-e2e-test resourceVersion: "15193" uid: 8a283ad4-dafc-495e-9366-afdeeb1dcf62 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7 namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: 505acb2b-854c-4ba7-a80f-68509f5fee55 kind: Event lastTimestamp: "2026-04-25T00:06:02Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:72e9a53b4f6609f050c864ccf8bba3bc53b15bbbb0a09e5de74a3bdc7465dd84" in 3.098s (3.098s including waiting). Image size: 301492132 bytes.' metadata: creationTimestamp: "2026-04-25T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:06:02Z" name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7.18a970cefea44331 namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: ee454b74-b083-4e5b-a9b6-5d4ba92c5fc2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7 namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: 505acb2b-854c-4ba7-a80f-68509f5fee55 kind: Event lastTimestamp: "2026-04-25T00:06:02Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-25T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:06:02Z" name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7.18a970cf01b9f954 namespace: kserve-ci-e2e-test resourceVersion: "15226" uid: 6467cd22-f832-4515-a799-14ed4ce028d1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7 namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: 505acb2b-854c-4ba7-a80f-68509f5fee55 kind: Event lastTimestamp: "2026-04-25T00:06:02Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-25T00:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:06:02Z" name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7.18a970cf023112d3 namespace: kserve-ci-e2e-test resourceVersion: "15227" uid: d0244bc5-9c69-4ff9-8595-172a9c906760 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:06:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7 namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: 505acb2b-854c-4ba7-a80f-68509f5fee55 kind: Event lastTimestamp: "2026-04-25T00:06:06Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-25T00:06:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:06:06Z" name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7.18a970cffdd96afe namespace: kserve-ci-e2e-test resourceVersion: "15279" uid: 6bcb083d-72a3-490e-9373-25cc695a82d9 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:06:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7 namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: 505acb2b-854c-4ba7-a80f-68509f5fee55 kind: Event lastTimestamp: "2026-04-25T00:06:25Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 19.251s (19.251s including waiting). Image size: 1306417402 bytes.' metadata: creationTimestamp: "2026-04-25T00:06:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:06:25Z" name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7.18a970d4795690ce namespace: kserve-ci-e2e-test resourceVersion: "15434" uid: 7a3db96c-c272-4b80-9d75-b74b0f543bbc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:06:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7 namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: 505acb2b-854c-4ba7-a80f-68509f5fee55 kind: Event lastTimestamp: "2026-04-25T00:06:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-25T00:06:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:06:25Z" name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7.18a970d47c6bd515 namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: be7da2fc-8b49-46c9-95c9-417f7248669f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:06:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7 namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: 505acb2b-854c-4ba7-a80f-68509f5fee55 kind: Event lastTimestamp: "2026-04-25T00:06:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-25T00:06:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:06:25Z" name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7.18a970d47cdc4ca2 namespace: kserve-ci-e2e-test resourceVersion: "15436" uid: 48a90a4e-ee16-4cce-93b5-a24118000026 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:06:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7 namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: 505acb2b-854c-4ba7-a80f-68509f5fee55 kind: Event lastTimestamp: "2026-04-25T00:06:25Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-25T00:06:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:06:25Z" name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7.18a970d47ce83372 namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: 35a8dad2-29e9-4ec9-bd42-fa73a6862af6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:06:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7 namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: 505acb2b-854c-4ba7-a80f-68509f5fee55 kind: Event lastTimestamp: "2026-04-25T00:06:28Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.292s (2.292s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-25T00:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:06:28Z" name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7.18a970d5058d827f namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: 5b76226b-d3f2-40d6-ac2a-ad01a0da82ba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:06:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7 namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: 505acb2b-854c-4ba7-a80f-68509f5fee55 kind: Event lastTimestamp: "2026-04-25T00:06:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-25T00:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:06:28Z" name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7.18a970d5084dddfc namespace: kserve-ci-e2e-test resourceVersion: "15453" uid: cbe8f0e8-22c9-4d54-bee0-8add310ffe2a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:06:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7 namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: 505acb2b-854c-4ba7-a80f-68509f5fee55 kind: Event lastTimestamp: "2026-04-25T00:06:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-25T00:06:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:06:28Z" name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7.18a970d508c9323c namespace: kserve-ci-e2e-test resourceVersion: "15454" uid: 2758b6db-fc8b-4645-b574-38fd4a6de6b2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-25T00:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7 namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: 505acb2b-854c-4ba7-a80f-68509f5fee55 kind: Event lastTimestamp: "2026-04-25T00:07:25Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-25T00:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:25Z" name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7.18a970d55d9ba1b7 namespace: kserve-ci-e2e-test resourceVersion: "15898" uid: 66319457-c141-4056-9a8e-11cf382b28c2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7 namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: 505acb2b-854c-4ba7-a80f-68509f5fee55 kind: Event lastTimestamp: "2026-04-25T00:07:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-25T00:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:49Z" name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7.18a970e7db8a8464 namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: c8c58655-a5e6-4f6d-81c0-7b2fb1a3ffa2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7 namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: 505acb2b-854c-4ba7-a80f-68509f5fee55 kind: Event lastTimestamp: "2026-04-25T00:07:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-25T00:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:49Z" name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7.18a970e7db8b1416 namespace: kserve-ci-e2e-test resourceVersion: "16221" uid: 2aa6af64-72e6-431a-9f53-92f8830e05c3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7 namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: 505acb2b-854c-4ba7-a80f-68509f5fee55 kind: Event lastTimestamp: "2026-04-25T00:07:50Z" message: 'Readiness probe failed: Get "https://10.132.0.25:8643/healthz": dial tcp 10.132.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-25T00:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:50Z" name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7.18a970e839bc8b3a namespace: kserve-ci-e2e-test resourceVersion: "16271" uid: 5d9fe528-8371-48f5-91c7-a51ac372efc1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:05:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49 namespace: kserve-ci-e2e-test resourceVersion: "15152" uid: ec7cbcff-d0c4-4b73-965e-35eea037e691 kind: Event lastTimestamp: "2026-04-25T00:05:58Z" message: 'Created pod: isvc-xgboost-graph-raw-3b499-predictor-f94984f49-pzpv7' metadata: creationTimestamp: "2026-04-25T00:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:05:58Z" name: isvc-xgboost-graph-raw-3b499-predictor-f94984f49.18a970ce2a616138 namespace: kserve-ci-e2e-test resourceVersion: "15165" uid: 3e21b570-0a31-4a89-9c9c-22386be35c2e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:05:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-3b499-predictor namespace: kserve-ci-e2e-test resourceVersion: "15146" uid: 7cba073b-c10c-4535-aa3f-1ce5eebd05b5 kind: Event lastTimestamp: "2026-04-25T00:05:58Z" message: Scaled up replica set isvc-xgboost-graph-raw-3b499-predictor-f94984f49 from 0 to 1 metadata: creationTimestamp: "2026-04-25T00:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:05:58Z" name: isvc-xgboost-graph-raw-3b499-predictor.18a970ce27669cdb namespace: kserve-ci-e2e-test resourceVersion: "15154" uid: 18e1a2c7-330d-451a-a332-6861fe717d9a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-25T00:06:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-3b499-predictor namespace: kserve-ci-e2e-test resourceVersion: "15151" uid: 74241a8e-e513-49df-8f8b-de5cb95b992e kind: Event lastTimestamp: "2026-04-25T00:06:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-25T00:06:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:06:58Z" name: isvc-xgboost-graph-raw-3b499-predictor.18a970d1a607802e namespace: kserve-ci-e2e-test resourceVersion: "15694" uid: d3495701-5424-4529-8d0e-87a90f5a5afa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-25T00:06:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-3b499-predictor namespace: kserve-ci-e2e-test resourceVersion: "15151" uid: 74241a8e-e513-49df-8f8b-de5cb95b992e kind: Event lastTimestamp: "2026-04-25T00:06:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-25T00:06:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:06:58Z" name: isvc-xgboost-graph-raw-3b499-predictor.18a970d1a6699e4c namespace: kserve-ci-e2e-test resourceVersion: "15695" uid: 923f8dd2-ef0e-41ae-975e-0bd110bf3724 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-25T00:06:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-3b499-predictor namespace: kserve-ci-e2e-test resourceVersion: "15342" uid: 74241a8e-e513-49df-8f8b-de5cb95b992e kind: Event lastTimestamp: "2026-04-25T00:07:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-25T00:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:07:43Z" name: isvc-xgboost-graph-raw-3b499-predictor.18a970d8a3d4b062 namespace: kserve-ci-e2e-test resourceVersion: "16075" uid: 6ef657ab-f1d9-466e-8c3f-214c0dcddd52 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-25T00:06:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-3b499-predictor namespace: kserve-ci-e2e-test resourceVersion: "15342" uid: 74241a8e-e513-49df-8f8b-de5cb95b992e kind: Event lastTimestamp: "2026-04-25T00:07:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-25T00:06:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:07:43Z" name: isvc-xgboost-graph-raw-3b499-predictor.18a970d8a4351377 namespace: kserve-ci-e2e-test resourceVersion: "16076" uid: a0bcfa63-9a23-4e31-b859-350d4d07f80f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-25T00:05:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-3b499 namespace: kserve-ci-e2e-test resourceVersion: "15133" uid: 5ed2d758-d29c-4b4f-acfa-5d2ef347c682 kind: Event lastTimestamp: "2026-04-25T00:07:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-25T00:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:07:35Z" name: isvc-xgboost-graph-raw-3b499.18a970ce276c7c62 namespace: kserve-ci-e2e-test resourceVersion: "15979" uid: e178fca4-41f7-4702-8190-a985b7054834 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-3b499 namespace: kserve-ci-e2e-test resourceVersion: "15977" uid: 5ed2d758-d29c-4b4f-acfa-5d2ef347c682 kind: Event lastTimestamp: "2026-04-25T00:07:35Z" message: InferenceService [isvc-xgboost-graph-raw-3b499] is Ready metadata: creationTimestamp: "2026-04-25T00:07:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:07:35Z" name: isvc-xgboost-graph-raw-3b499.18a970e4c06a1230 namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: f91927de-0aac-41b8-9e28-5b2390fc4e80 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-25T00:07:49.119265Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb namespace: kserve-ci-e2e-test resourceVersion: "16223" uid: 4d9c96ff-0e6a-4265-8a48-450f1fdf5501 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb to ip-10-0-139-62.ec2.internal metadata: creationTimestamp: "2026-04-25T00:07:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-25T00:07:49Z" name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb.18a970e7dcd80a5f namespace: kserve-ci-e2e-test resourceVersion: "16227" uid: bdfb4efb-0157-41aa-aff0-a76a435b051c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56c7745b65-967mt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:49Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: 4d9c96ff-0e6a-4265-8a48-450f1fdf5501 kind: Event lastTimestamp: "2026-04-25T00:07:49Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-25T00:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-25T00:07:49Z" name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb.18a970e7f6be7ea3 namespace: kserve-ci-e2e-test resourceVersion: "16242" uid: cd07b3b6-c8c4-4e20-8619-cce928696f67 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb namespace: kserve-ci-e2e-test resourceVersion: "16225" uid: 4d9c96ff-0e6a-4265-8a48-450f1fdf5501 kind: Event lastTimestamp: "2026-04-25T00:07:49Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:72e9a53b4f6609f050c864ccf8bba3bc53b15bbbb0a09e5de74a3bdc7465dd84" already present on machine metadata: creationTimestamp: "2026-04-25T00:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:49Z" name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb.18a970e7f84d6d12 namespace: kserve-ci-e2e-test resourceVersion: "16245" uid: 61b8d934-0b43-4e03-bbcf-eeca74a4cca0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb namespace: kserve-ci-e2e-test resourceVersion: "16225" uid: 4d9c96ff-0e6a-4265-8a48-450f1fdf5501 kind: Event lastTimestamp: "2026-04-25T00:07:49Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-25T00:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:49Z" name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb.18a970e7fb173ac5 namespace: kserve-ci-e2e-test resourceVersion: "16248" uid: 008a93d0-17d7-4039-b7af-e4efef03a87c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb namespace: kserve-ci-e2e-test resourceVersion: "16225" uid: 4d9c96ff-0e6a-4265-8a48-450f1fdf5501 kind: Event lastTimestamp: "2026-04-25T00:07:49Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-25T00:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:49Z" name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb.18a970e7fb980171 namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: 9ebea6ad-a163-467b-b7b4-8992ff3976ae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb namespace: kserve-ci-e2e-test resourceVersion: "16225" uid: 4d9c96ff-0e6a-4265-8a48-450f1fdf5501 kind: Event lastTimestamp: "2026-04-25T00:07:53Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-04-25T00:07:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:53Z" name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb.18a970e8fd2c0e13 namespace: kserve-ci-e2e-test resourceVersion: "16306" uid: 98436ee4-e196-46ca-81b9-9c8142ab6173 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb namespace: kserve-ci-e2e-test resourceVersion: "16225" uid: 4d9c96ff-0e6a-4265-8a48-450f1fdf5501 kind: Event lastTimestamp: "2026-04-25T00:07:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-25T00:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:54Z" name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb.18a970e900b86b04 namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: c7515472-b701-4174-8f57-271a50e3b809 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb namespace: kserve-ci-e2e-test resourceVersion: "16225" uid: 4d9c96ff-0e6a-4265-8a48-450f1fdf5501 kind: Event lastTimestamp: "2026-04-25T00:07:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-25T00:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:54Z" name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb.18a970e9013cb58d namespace: kserve-ci-e2e-test resourceVersion: "16313" uid: 0e1d6a21-f20e-4b3f-9b54-d5af9924484d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb namespace: kserve-ci-e2e-test resourceVersion: "16225" uid: 4d9c96ff-0e6a-4265-8a48-450f1fdf5501 kind: Event lastTimestamp: "2026-04-25T00:07:54Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-25T00:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:54Z" name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb.18a970e90149fba1 namespace: kserve-ci-e2e-test resourceVersion: "16314" uid: 9d3f1db6-ab7f-406e-946f-88379350a815 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb namespace: kserve-ci-e2e-test resourceVersion: "16225" uid: 4d9c96ff-0e6a-4265-8a48-450f1fdf5501 kind: Event lastTimestamp: "2026-04-25T00:07:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-25T00:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:54Z" name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb.18a970e904665b8a namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: 27aa45c5-b97d-4173-a327-ccf1b08e2566 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb namespace: kserve-ci-e2e-test resourceVersion: "16225" uid: 4d9c96ff-0e6a-4265-8a48-450f1fdf5501 kind: Event lastTimestamp: "2026-04-25T00:07:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-25T00:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:54Z" name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb.18a970e904e397a7 namespace: kserve-ci-e2e-test resourceVersion: "16318" uid: 14b842f9-2ef9-401b-86ad-8dd65c38342d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-25T00:07:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb namespace: kserve-ci-e2e-test resourceVersion: "16225" uid: 4d9c96ff-0e6a-4265-8a48-450f1fdf5501 kind: Event lastTimestamp: "2026-04-25T00:09:30Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-25T00:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:30Z" name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb.18a970e939264bd3 namespace: kserve-ci-e2e-test resourceVersion: "17212" uid: 026e5a71-54e8-4bb8-8464-6ca636a6647f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb namespace: kserve-ci-e2e-test resourceVersion: "16225" uid: 4d9c96ff-0e6a-4265-8a48-450f1fdf5501 kind: Event lastTimestamp: "2026-04-25T00:09:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-25T00:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:29Z" name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb.18a970ff3252f550 namespace: kserve-ci-e2e-test resourceVersion: "17182" uid: aa9d320f-75cc-402e-93c5-442007b09e72 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb namespace: kserve-ci-e2e-test resourceVersion: "16225" uid: 4d9c96ff-0e6a-4265-8a48-450f1fdf5501 kind: Event lastTimestamp: "2026-04-25T00:09:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-25T00:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:29Z" name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb.18a970ff32540174 namespace: kserve-ci-e2e-test resourceVersion: "17183" uid: 40797d69-e627-44ba-869d-bd781eef73e0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb namespace: kserve-ci-e2e-test resourceVersion: "16225" uid: 4d9c96ff-0e6a-4265-8a48-450f1fdf5501 kind: Event lastTimestamp: "2026-04-25T00:09:30Z" message: 'Readiness probe failed: Get "https://10.132.0.26:8643/healthz": dial tcp 10.132.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-25T00:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:30Z" name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb.18a970ff93713f97 namespace: kserve-ci-e2e-test resourceVersion: "17210" uid: c97754c0-6d7a-4984-82ed-25dc611a3f18 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c namespace: kserve-ci-e2e-test resourceVersion: "16192" uid: b9a471ce-a1d8-4e4d-9f9a-5044236575b9 kind: Event lastTimestamp: "2026-04-25T00:07:49Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c-2s7tb' metadata: creationTimestamp: "2026-04-25T00:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:07:49Z" name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c.18a970e7dc7afe9c namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: 7da181d8-020e-42da-9b7f-d852724cc5d4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: 357b52c1-5513-4f00-8096-1b507ed15b31 kind: Event lastTimestamp: "2026-04-25T00:07:49Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-3f0c4-predictor-76f85bb77c from 0 to 1 metadata: creationTimestamp: "2026-04-25T00:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:07:49Z" name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor.18a970e7d6f78683 namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: 727da705-1499-4f9d-8809-765a799190eb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-25T00:08:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor namespace: kserve-ci-e2e-test resourceVersion: "16193" uid: 6450a2b8-04b1-4620-a58c-81a181448573 kind: Event lastTimestamp: "2026-04-25T00:08:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-25T00:08:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:08:34Z" name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor.18a970eb5593aa7a namespace: kserve-ci-e2e-test resourceVersion: "16648" uid: db178574-0e86-4e47-bc15-5dbccd55e4fd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-25T00:08:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor namespace: kserve-ci-e2e-test resourceVersion: "16193" uid: 6450a2b8-04b1-4620-a58c-81a181448573 kind: Event lastTimestamp: "2026-04-25T00:08:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-25T00:08:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:08:34Z" name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor.18a970eb55fab1b9 namespace: kserve-ci-e2e-test resourceVersion: "16649" uid: c9e2cf51-dc3a-4c01-8a6d-47b937094cca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-25T00:08:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor namespace: kserve-ci-e2e-test resourceVersion: "16415" uid: 6450a2b8-04b1-4620-a58c-81a181448573 kind: Event lastTimestamp: "2026-04-25T00:09:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-25T00:08:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:09:19Z" name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor.18a970eed4a7936c namespace: kserve-ci-e2e-test resourceVersion: "17016" uid: a7d1ce11-ff81-42f1-9bba-68607d19b653 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-25T00:08:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor namespace: kserve-ci-e2e-test resourceVersion: "16415" uid: 6450a2b8-04b1-4620-a58c-81a181448573 kind: Event lastTimestamp: "2026-04-25T00:09:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-25T00:08:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:09:19Z" name: isvc-xgboost-graph-raw-hpa-3f0c4-predictor.18a970eed51b8e1f namespace: kserve-ci-e2e-test resourceVersion: "17017" uid: 9934205b-0e09-462d-b255-cb4bd0d9a6ec reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-25T00:07:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-3f0c4 namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: 923be306-4509-4746-b804-024d4d901301 kind: Event lastTimestamp: "2026-04-25T00:09:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-25T00:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:09:01Z" name: isvc-xgboost-graph-raw-hpa-3f0c4.18a970e7d701c124 namespace: kserve-ci-e2e-test resourceVersion: "16851" uid: 2c1944d7-63d2-43ba-8370-33a1a4b364f8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-3f0c4 namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: 923be306-4509-4746-b804-024d4d901301 kind: Event lastTimestamp: "2026-04-25T00:07:49Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-hpa-3f0c4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-hpa-3f0c4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-25T00:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:07:49Z" name: isvc-xgboost-graph-raw-hpa-3f0c4.18a970e7d78b1eec namespace: kserve-ci-e2e-test resourceVersion: "16197" uid: 446d91b2-a7ed-449b-a207-0fb6b6ecb786 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-3f0c4 namespace: kserve-ci-e2e-test resourceVersion: "16849" uid: 923be306-4509-4746-b804-024d4d901301 kind: Event lastTimestamp: "2026-04-25T00:09:01Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-3f0c4] is Ready metadata: creationTimestamp: "2026-04-25T00:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:09:01Z" name: isvc-xgboost-graph-raw-hpa-3f0c4.18a970f89c410182 namespace: kserve-ci-e2e-test resourceVersion: "16850" uid: a931e110-55e9-467f-ad24-2d1c3bdc9979 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-25T00:09:29.305204Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-b8788-predictor-85565b99f9-hckmm namespace: kserve-ci-e2e-test resourceVersion: "17157" uid: b8fc01df-c19f-4457-9b79-e1aaa951822d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-b8788-predictor-85565b99f9-hckmm to ip-10-0-139-62.ec2.internal metadata: creationTimestamp: "2026-04-25T00:09:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-25T00:09:29Z" name: message-dumper-raw-b8788-predictor-85565b99f9-hckmm.18a970ff30642d02 namespace: kserve-ci-e2e-test resourceVersion: "17167" uid: f646ac71-584a-49b5-8b88-c8701c20558b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56c7745b65-967mt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:29Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-b8788-predictor-85565b99f9-hckmm namespace: kserve-ci-e2e-test resourceVersion: "17181" uid: b8fc01df-c19f-4457-9b79-e1aaa951822d kind: Event lastTimestamp: "2026-04-25T00:09:29Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-25T00:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-25T00:09:29Z" name: message-dumper-raw-b8788-predictor-85565b99f9-hckmm.18a970ff49c9f758 namespace: kserve-ci-e2e-test resourceVersion: "17193" uid: 9c37289a-a646-4dd7-8606-163a8088a6c7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-b8788-predictor-85565b99f9-hckmm namespace: kserve-ci-e2e-test resourceVersion: "17165" uid: b8fc01df-c19f-4457-9b79-e1aaa951822d kind: Event lastTimestamp: "2026-04-25T00:09:29Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-04-25T00:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:29Z" name: message-dumper-raw-b8788-predictor-85565b99f9-hckmm.18a970ff4b0b00bb namespace: kserve-ci-e2e-test resourceVersion: "17195" uid: aed80331-1f76-4993-8daa-6457f358c4a6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-b8788-predictor-85565b99f9-hckmm namespace: kserve-ci-e2e-test resourceVersion: "17165" uid: b8fc01df-c19f-4457-9b79-e1aaa951822d kind: Event lastTimestamp: "2026-04-25T00:09:30Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.105s (1.105s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-04-25T00:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:30Z" name: message-dumper-raw-b8788-predictor-85565b99f9-hckmm.18a970ff8ce92f78 namespace: kserve-ci-e2e-test resourceVersion: "17206" uid: 88c8a098-fb40-41ed-8681-cf48fbaa921d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-b8788-predictor-85565b99f9-hckmm namespace: kserve-ci-e2e-test resourceVersion: "17165" uid: b8fc01df-c19f-4457-9b79-e1aaa951822d kind: Event lastTimestamp: "2026-04-25T00:09:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-25T00:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:30Z" name: message-dumper-raw-b8788-predictor-85565b99f9-hckmm.18a970ff8fba2538 namespace: kserve-ci-e2e-test resourceVersion: "17207" uid: 1963a5e4-0eb4-4eaf-b4fb-54183e46b46e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-b8788-predictor-85565b99f9-hckmm namespace: kserve-ci-e2e-test resourceVersion: "17165" uid: b8fc01df-c19f-4457-9b79-e1aaa951822d kind: Event lastTimestamp: "2026-04-25T00:09:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-25T00:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:30Z" name: message-dumper-raw-b8788-predictor-85565b99f9-hckmm.18a970ff9045cd9c namespace: kserve-ci-e2e-test resourceVersion: "17208" uid: 2331b249-87d7-47a6-8aac-d998edd76527 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-b8788-predictor-85565b99f9-hckmm namespace: kserve-ci-e2e-test resourceVersion: "17165" uid: b8fc01df-c19f-4457-9b79-e1aaa951822d kind: Event lastTimestamp: "2026-04-25T00:09:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-25T00:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:30Z" name: message-dumper-raw-b8788-predictor-85565b99f9-hckmm.18a970ff90534657 namespace: kserve-ci-e2e-test resourceVersion: "17209" uid: 3f5e4816-cc77-472d-89a0-2e5ea51afa0a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-b8788-predictor-85565b99f9-hckmm namespace: kserve-ci-e2e-test resourceVersion: "17165" uid: b8fc01df-c19f-4457-9b79-e1aaa951822d kind: Event lastTimestamp: "2026-04-25T00:09:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-25T00:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:30Z" name: message-dumper-raw-b8788-predictor-85565b99f9-hckmm.18a970ff9399c37a namespace: kserve-ci-e2e-test resourceVersion: "17211" uid: c2ddc943-5378-4bb9-b7e1-d359f19e3c19 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-b8788-predictor-85565b99f9-hckmm namespace: kserve-ci-e2e-test resourceVersion: "17165" uid: b8fc01df-c19f-4457-9b79-e1aaa951822d kind: Event lastTimestamp: "2026-04-25T00:09:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-25T00:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:30Z" name: message-dumper-raw-b8788-predictor-85565b99f9-hckmm.18a970ff9412b55f namespace: kserve-ci-e2e-test resourceVersion: "17213" uid: 3ed5feae-0b24-4543-b6c5-8ec7b0378e18 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:11:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-b8788-predictor-85565b99f9-hckmm namespace: kserve-ci-e2e-test resourceVersion: "17165" uid: b8fc01df-c19f-4457-9b79-e1aaa951822d kind: Event lastTimestamp: "2026-04-25T00:11:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-25T00:11:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:11:04Z" name: message-dumper-raw-b8788-predictor-85565b99f9-hckmm.18a9711563768827 namespace: kserve-ci-e2e-test resourceVersion: "18065" uid: ee8656bb-689e-4dc7-b026-b493e0c8c867 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:11:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-b8788-predictor-85565b99f9-hckmm namespace: kserve-ci-e2e-test resourceVersion: "17165" uid: b8fc01df-c19f-4457-9b79-e1aaa951822d kind: Event lastTimestamp: "2026-04-25T00:11:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-25T00:11:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:11:04Z" name: message-dumper-raw-b8788-predictor-85565b99f9-hckmm.18a971156376f211 namespace: kserve-ci-e2e-test resourceVersion: "18066" uid: 8f702a07-e61d-4bdc-9c9e-f99d236a767e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-62.ec2.internal source: component: kubelet host: ip-10-0-139-62.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-b8788-predictor-85565b99f9 namespace: kserve-ci-e2e-test resourceVersion: "17140" uid: 74e5b59b-b3eb-458e-8eab-aead73f8eba7 kind: Event lastTimestamp: "2026-04-25T00:09:29Z" message: 'Created pod: message-dumper-raw-b8788-predictor-85565b99f9-hckmm' metadata: creationTimestamp: "2026-04-25T00:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:09:29Z" name: message-dumper-raw-b8788-predictor-85565b99f9.18a970ff2ff628cb namespace: kserve-ci-e2e-test resourceVersion: "17164" uid: d9a4bc73-1431-47e4-8281-267bf4b74c5c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-b8788-predictor namespace: kserve-ci-e2e-test resourceVersion: "17134" uid: cb2537e1-ed35-46b0-9be8-a44e2286f3f1 kind: Event lastTimestamp: "2026-04-25T00:09:29Z" message: Scaled up replica set message-dumper-raw-b8788-predictor-85565b99f9 from 0 to 1 metadata: creationTimestamp: "2026-04-25T00:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:09:29Z" name: message-dumper-raw-b8788-predictor.18a970ff2c74a455 namespace: kserve-ci-e2e-test resourceVersion: "17146" uid: 07afb19c-e9c4-48d8-a2c7-754ab44966b5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-25T00:09:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-b8788-predictor namespace: kserve-ci-e2e-test resourceVersion: "17139" uid: 589ca308-e8a3-44aa-a67d-2b8f8d5edde4 kind: Event lastTimestamp: "2026-04-25T00:09:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-25T00:09:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:09:59Z" name: message-dumper-raw-b8788-predictor.18a97102aade866a namespace: kserve-ci-e2e-test resourceVersion: "17511" uid: e24eaee2-7dfa-41be-a96e-9ef47938dd4a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-25T00:09:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-b8788-predictor namespace: kserve-ci-e2e-test resourceVersion: "17139" uid: 589ca308-e8a3-44aa-a67d-2b8f8d5edde4 kind: Event lastTimestamp: "2026-04-25T00:09:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-25T00:09:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:09:59Z" name: message-dumper-raw-b8788-predictor.18a97102ab344d4b namespace: kserve-ci-e2e-test resourceVersion: "17512" uid: 1b14610d-ffb5-47cc-8884-3eedde53ec96 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-25T00:09:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-b8788 namespace: kserve-ci-e2e-test resourceVersion: "17120" uid: 56432f0e-bb2a-4728-b427-6cec66e123c7 kind: Event lastTimestamp: "2026-04-25T00:09:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-25T00:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:09:38Z" name: message-dumper-raw-b8788.18a970ff2c52dcfd namespace: kserve-ci-e2e-test resourceVersion: "17287" uid: f683d292-150e-43bf-ba80-f00a45a59f41 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-b8788 namespace: kserve-ci-e2e-test resourceVersion: "17120" uid: 56432f0e-bb2a-4728-b427-6cec66e123c7 kind: Event lastTimestamp: "2026-04-25T00:09:29Z" message: 'Failed to update status for InferenceService "message-dumper-raw-b8788": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-b8788": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-25T00:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:09:29Z" name: message-dumper-raw-b8788.18a970ff2ceab182 namespace: kserve-ci-e2e-test resourceVersion: "17150" uid: ce2c8340-14bb-4bfb-ab20-2896f3486b7d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-b8788 namespace: kserve-ci-e2e-test resourceVersion: "17284" uid: 56432f0e-bb2a-4728-b427-6cec66e123c7 kind: Event lastTimestamp: "2026-04-25T00:09:38Z" message: InferenceService [message-dumper-raw-b8788] is Ready metadata: creationTimestamp: "2026-04-25T00:09:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:09:38Z" name: message-dumper-raw-b8788.18a971014c24fdad namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 34a0a821-1f75-4109-83bb-f66218211f6b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-25T00:07:38.729808Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-3b499-57f97cb5b9-zc5fk namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: 98cd8436-3894-46a1-8eff-e943920e9581 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-3b499-57f97cb5b9-zc5fk to ip-10-0-132-64.ec2.internal metadata: creationTimestamp: "2026-04-25T00:07:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-25T00:07:38Z" name: model-chainer-raw-3b499-57f97cb5b9-zc5fk.18a970e5719581fe namespace: kserve-ci-e2e-test resourceVersion: "16018" uid: 17575def-71f8-466d-a3d0-29f86e54da9e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56c7745b65-967mt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:39Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-3b499-57f97cb5b9-zc5fk namespace: kserve-ci-e2e-test resourceVersion: "16022" uid: 98cd8436-3894-46a1-8eff-e943920e9581 kind: Event lastTimestamp: "2026-04-25T00:07:39Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-25T00:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-25T00:07:39Z" name: model-chainer-raw-3b499-57f97cb5b9-zc5fk.18a970e58afec166 namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: ac7a2f0f-d392-4c3d-a84c-6d6b39592129 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-3b499} kind: Pod name: model-chainer-raw-3b499-57f97cb5b9-zc5fk namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 98cd8436-3894-46a1-8eff-e943920e9581 kind: Event lastTimestamp: "2026-04-25T00:07:39Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:8608123ddb1fc42bff2e1c9ce408ce175b3696948e68bd4ec98a77fff89c49f4" metadata: creationTimestamp: "2026-04-25T00:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:39Z" name: model-chainer-raw-3b499-57f97cb5b9-zc5fk.18a970e58c4a67cb namespace: kserve-ci-e2e-test resourceVersion: "16037" uid: 610c8ed0-373d-4b7e-a956-f27853d48c56 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-3b499} kind: Pod name: model-chainer-raw-3b499-57f97cb5b9-zc5fk namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 98cd8436-3894-46a1-8eff-e943920e9581 kind: Event lastTimestamp: "2026-04-25T00:07:41Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:8608123ddb1fc42bff2e1c9ce408ce175b3696948e68bd4ec98a77fff89c49f4" in 2.335s (2.336s including waiting). Image size: 216346185 bytes.' metadata: creationTimestamp: "2026-04-25T00:07:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:41Z" name: model-chainer-raw-3b499-57f97cb5b9-zc5fk.18a970e617875d49 namespace: kserve-ci-e2e-test resourceVersion: "16051" uid: fb95c766-383c-410c-9bfd-87d84efb44fb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-3b499} kind: Pod name: model-chainer-raw-3b499-57f97cb5b9-zc5fk namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 98cd8436-3894-46a1-8eff-e943920e9581 kind: Event lastTimestamp: "2026-04-25T00:07:41Z" message: 'Created container: model-chainer-raw-3b499' metadata: creationTimestamp: "2026-04-25T00:07:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:41Z" name: model-chainer-raw-3b499-57f97cb5b9-zc5fk.18a970e61adbe95f namespace: kserve-ci-e2e-test resourceVersion: "16054" uid: 906f54f2-6e70-4ec0-ab35-b174bf740863 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-3b499} kind: Pod name: model-chainer-raw-3b499-57f97cb5b9-zc5fk namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 98cd8436-3894-46a1-8eff-e943920e9581 kind: Event lastTimestamp: "2026-04-25T00:07:41Z" message: Started container model-chainer-raw-3b499 metadata: creationTimestamp: "2026-04-25T00:07:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:41Z" name: model-chainer-raw-3b499-57f97cb5b9-zc5fk.18a970e61b504184 namespace: kserve-ci-e2e-test resourceVersion: "16055" uid: dd0cd1c9-b4d7-44d7-8de2-79c27f1ccad4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-3b499} kind: Pod name: model-chainer-raw-3b499-57f97cb5b9-zc5fk namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 98cd8436-3894-46a1-8eff-e943920e9581 kind: Event lastTimestamp: "2026-04-25T00:07:48Z" message: Stopping container model-chainer-raw-3b499 metadata: creationTimestamp: "2026-04-25T00:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:07:48Z" name: model-chainer-raw-3b499-57f97cb5b9-zc5fk.18a970e7c9fe07c7 namespace: kserve-ci-e2e-test resourceVersion: "16130" uid: 0cfd631c-1d2a-44ee-86e1-5744f1cbf551 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-25T00:07:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-3b499} kind: Pod name: model-chainer-raw-3b499-57f97cb5b9-zc5fk namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 98cd8436-3894-46a1-8eff-e943920e9581 kind: Event lastTimestamp: "2026-04-25T00:08:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-25T00:07:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:08:18Z" name: model-chainer-raw-3b499-57f97cb5b9-zc5fk.18a970e8da75a3a8 namespace: kserve-ci-e2e-test resourceVersion: "16518" uid: 87adbdd2-f1be-405c-8a48-b111ba9095b6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-3b499-57f97cb5b9 namespace: kserve-ci-e2e-test resourceVersion: "16009" uid: 258be06f-cf16-4bcf-be85-48d8c9aecef3 kind: Event lastTimestamp: "2026-04-25T00:07:38Z" message: 'Created pod: model-chainer-raw-3b499-57f97cb5b9-zc5fk' metadata: creationTimestamp: "2026-04-25T00:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:07:38Z" name: model-chainer-raw-3b499-57f97cb5b9.18a970e5712ea855 namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: ba51995c-0cf7-4c17-987b-53b5c8ee1b23 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-3b499 namespace: kserve-ci-e2e-test resourceVersion: "16007" uid: 037d0e8b-ad8f-470d-b987-441d551d5627 kind: Event lastTimestamp: "2026-04-25T00:07:38Z" message: Scaled up replica set model-chainer-raw-3b499-57f97cb5b9 from 0 to 1 metadata: creationTimestamp: "2026-04-25T00:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:07:38Z" name: model-chainer-raw-3b499.18a970e56fa73f78 namespace: kserve-ci-e2e-test resourceVersion: "16010" uid: 891a1a9d-f7c0-45e8-9cb2-48e3a610c2d9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:07:38Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-3b499 namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 4b921063-6812-47c5-a265-d16077861972 kind: Event lastTimestamp: "2026-04-25T00:07:38Z" message: InferenceGraph [model-chainer-raw-3b499] is Ready metadata: creationTimestamp: "2026-04-25T00:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:07:38Z" name: model-chainer-raw-3b499.18a970e57593fab2 namespace: kserve-ci-e2e-test resourceVersion: "16029" uid: 6c9886ec-9343-40c4-94e9-d4a522f87330 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-25T00:09:19.046400Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-3f0c4-868c7dd7d5-g9vg8 namespace: kserve-ci-e2e-test resourceVersion: "17003" uid: 3a73f646-e3e6-4acc-a61e-c5317041365a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-3f0c4-868c7dd7d5-g9vg8 to ip-10-0-132-64.ec2.internal metadata: creationTimestamp: "2026-04-25T00:09:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-25T00:09:19Z" name: model-chainer-raw-hpa-3f0c4-868c7dd7d5-g9vg8.18a970fccceb2c06 namespace: kserve-ci-e2e-test resourceVersion: "17007" uid: cede12f1-cacf-48f0-b715-e8b6afd3bd23 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56c7745b65-967mt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:19Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-3f0c4-868c7dd7d5-g9vg8 namespace: kserve-ci-e2e-test resourceVersion: "17011" uid: 3a73f646-e3e6-4acc-a61e-c5317041365a kind: Event lastTimestamp: "2026-04-25T00:09:19Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-25T00:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-25T00:09:19Z" name: model-chainer-raw-hpa-3f0c4-868c7dd7d5-g9vg8.18a970fce6624788 namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 08bb7a43-e81f-4601-82eb-560fe0dbdacd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3f0c4} kind: Pod name: model-chainer-raw-hpa-3f0c4-868c7dd7d5-g9vg8 namespace: kserve-ci-e2e-test resourceVersion: "17006" uid: 3a73f646-e3e6-4acc-a61e-c5317041365a kind: Event lastTimestamp: "2026-04-25T00:09:19Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:8608123ddb1fc42bff2e1c9ce408ce175b3696948e68bd4ec98a77fff89c49f4" already present on machine metadata: creationTimestamp: "2026-04-25T00:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:19Z" name: model-chainer-raw-hpa-3f0c4-868c7dd7d5-g9vg8.18a970fce797e0ff namespace: kserve-ci-e2e-test resourceVersion: "17027" uid: 15b2cffd-20e5-4693-999f-6a7ccb349feb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3f0c4} kind: Pod name: model-chainer-raw-hpa-3f0c4-868c7dd7d5-g9vg8 namespace: kserve-ci-e2e-test resourceVersion: "17006" uid: 3a73f646-e3e6-4acc-a61e-c5317041365a kind: Event lastTimestamp: "2026-04-25T00:09:19Z" message: 'Created container: model-chainer-raw-hpa-3f0c4' metadata: creationTimestamp: "2026-04-25T00:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:19Z" name: model-chainer-raw-hpa-3f0c4-868c7dd7d5-g9vg8.18a970fceadce226 namespace: kserve-ci-e2e-test resourceVersion: "17029" uid: e05dc2ef-53ec-4228-9af9-232e4b8114db reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3f0c4} kind: Pod name: model-chainer-raw-hpa-3f0c4-868c7dd7d5-g9vg8 namespace: kserve-ci-e2e-test resourceVersion: "17006" uid: 3a73f646-e3e6-4acc-a61e-c5317041365a kind: Event lastTimestamp: "2026-04-25T00:09:19Z" message: Started container model-chainer-raw-hpa-3f0c4 metadata: creationTimestamp: "2026-04-25T00:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:19Z" name: model-chainer-raw-hpa-3f0c4-868c7dd7d5-g9vg8.18a970fceb5696fa namespace: kserve-ci-e2e-test resourceVersion: "17030" uid: a36c439d-c0bd-4963-ba9a-63f1cf234ea6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3f0c4} kind: Pod name: model-chainer-raw-hpa-3f0c4-868c7dd7d5-g9vg8 namespace: kserve-ci-e2e-test resourceVersion: "17006" uid: 3a73f646-e3e6-4acc-a61e-c5317041365a kind: Event lastTimestamp: "2026-04-25T00:09:29Z" message: Stopping container model-chainer-raw-hpa-3f0c4 metadata: creationTimestamp: "2026-04-25T00:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:29Z" name: model-chainer-raw-hpa-3f0c4-868c7dd7d5-g9vg8.18a970ff23f4c18d namespace: kserve-ci-e2e-test resourceVersion: "17115" uid: 5e487f1d-716c-43ff-b7b9-587710f8e965 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-25T00:09:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3f0c4} kind: Pod name: model-chainer-raw-hpa-3f0c4-868c7dd7d5-g9vg8 namespace: kserve-ci-e2e-test resourceVersion: "17006" uid: 3a73f646-e3e6-4acc-a61e-c5317041365a kind: Event lastTimestamp: "2026-04-25T00:09:55Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-25T00:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:09:55Z" name: model-chainer-raw-hpa-3f0c4-868c7dd7d5-g9vg8.18a970ff8594951e namespace: kserve-ci-e2e-test resourceVersion: "17487" uid: e6825969-f58d-4802-84cb-37878a040a55 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-3f0c4-868c7dd7d5 namespace: kserve-ci-e2e-test resourceVersion: "16997" uid: 5f5f1271-c4ae-4ca6-a2dd-754244d8f431 kind: Event lastTimestamp: "2026-04-25T00:09:19Z" message: 'Created pod: model-chainer-raw-hpa-3f0c4-868c7dd7d5-g9vg8' metadata: creationTimestamp: "2026-04-25T00:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:09:19Z" name: model-chainer-raw-hpa-3f0c4-868c7dd7d5.18a970fccc636bc2 namespace: kserve-ci-e2e-test resourceVersion: "17004" uid: c967ed3d-df6a-4ed3-be36-882163e044de reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-3f0c4 namespace: kserve-ci-e2e-test resourceVersion: "16995" uid: 77126ce9-9462-4dd5-82fc-324cfdd53815 kind: Event lastTimestamp: "2026-04-25T00:09:19Z" message: Scaled up replica set model-chainer-raw-hpa-3f0c4-868c7dd7d5 from 0 to 1 metadata: creationTimestamp: "2026-04-25T00:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:09:19Z" name: model-chainer-raw-hpa-3f0c4.18a970fccb23a0ff namespace: kserve-ci-e2e-test resourceVersion: "16998" uid: c99002f5-0db7-4272-b80a-d9ccc9d1599a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:09:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-3f0c4 namespace: kserve-ci-e2e-test resourceVersion: "17013" uid: 321fc660-af04-4e13-896c-dd1a15544d35 kind: Event lastTimestamp: "2026-04-25T00:09:19Z" message: InferenceGraph [model-chainer-raw-hpa-3f0c4] is Ready metadata: creationTimestamp: "2026-04-25T00:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:09:19Z" name: model-chainer-raw-hpa-3f0c4.18a970fccf18f749 namespace: kserve-ci-e2e-test resourceVersion: "17015" uid: 8a7addb7-91da-4919-8546-2b3b18c2bcdf reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-25T00:15:31.407410Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: 69a9a267-5518-4d44-9461-fe6d2793a548 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-926cf-predictor-6fc8899c49-49qgj to ip-10-0-132-64.ec2.internal metadata: creationTimestamp: "2026-04-25T00:15:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-25T00:15:31Z" name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj.18a971537f5d4cc2 namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: 6ea221c7-e174-48c6-aa1b-a3a849869dc2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56c7745b65-967mt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:31Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: 69a9a267-5518-4d44-9461-fe6d2793a548 kind: Event lastTimestamp: "2026-04-25T00:15:31Z" message: Add eth0 [10.134.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-25T00:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-25T00:15:31Z" name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj.18a971539950320a namespace: kserve-ci-e2e-test resourceVersion: "20344" uid: dd7ac646-4cf8-45d8-92bd-56b9ad7bd9e4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj namespace: kserve-ci-e2e-test resourceVersion: "20325" uid: 69a9a267-5518-4d44-9461-fe6d2793a548 kind: Event lastTimestamp: "2026-04-25T00:15:31Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:72e9a53b4f6609f050c864ccf8bba3bc53b15bbbb0a09e5de74a3bdc7465dd84" already present on machine metadata: creationTimestamp: "2026-04-25T00:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:15:31Z" name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj.18a971539aaa0be9 namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: 3f934821-a67d-4e00-8348-7637d53905f0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj namespace: kserve-ci-e2e-test resourceVersion: "20325" uid: 69a9a267-5518-4d44-9461-fe6d2793a548 kind: Event lastTimestamp: "2026-04-25T00:15:31Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-25T00:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:15:31Z" name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj.18a971539e749e73 namespace: kserve-ci-e2e-test resourceVersion: "20348" uid: 96aecae5-6387-45ae-b527-4581bb83fe0a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj namespace: kserve-ci-e2e-test resourceVersion: "20325" uid: 69a9a267-5518-4d44-9461-fe6d2793a548 kind: Event lastTimestamp: "2026-04-25T00:15:31Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-25T00:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:15:31Z" name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj.18a971539eedf323 namespace: kserve-ci-e2e-test resourceVersion: "20349" uid: b8d3c740-e14f-49d1-be4c-f7dce29e1ec3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj namespace: kserve-ci-e2e-test resourceVersion: "20325" uid: 69a9a267-5518-4d44-9461-fe6d2793a548 kind: Event lastTimestamp: "2026-04-25T00:15:36Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1435" already present on machine metadata: creationTimestamp: "2026-04-25T00:15:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:15:36Z" name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj.18a9715499c4bad8 namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: ae75024b-e13e-42b1-9fd7-177b22a9190d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj namespace: kserve-ci-e2e-test resourceVersion: "20325" uid: 69a9a267-5518-4d44-9461-fe6d2793a548 kind: Event lastTimestamp: "2026-04-25T00:15:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-25T00:15:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:15:36Z" name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj.18a971549d1de324 namespace: kserve-ci-e2e-test resourceVersion: "20391" uid: 26471de2-933c-4c83-9daa-6c10e7591971 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj namespace: kserve-ci-e2e-test resourceVersion: "20325" uid: 69a9a267-5518-4d44-9461-fe6d2793a548 kind: Event lastTimestamp: "2026-04-25T00:15:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-25T00:15:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:15:36Z" name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj.18a971549d987586 namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 0b69f7b0-b38e-4e88-93d9-d4d4d8534c65 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj namespace: kserve-ci-e2e-test resourceVersion: "20325" uid: 69a9a267-5518-4d44-9461-fe6d2793a548 kind: Event lastTimestamp: "2026-04-25T00:15:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-25T00:15:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:15:36Z" name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj.18a971549da4f745 namespace: kserve-ci-e2e-test resourceVersion: "20393" uid: e81ac845-2022-4761-9f84-a1cd98a25f0c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj namespace: kserve-ci-e2e-test resourceVersion: "20325" uid: 69a9a267-5518-4d44-9461-fe6d2793a548 kind: Event lastTimestamp: "2026-04-25T00:15:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-25T00:15:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:15:36Z" name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj.18a97154a10b8237 namespace: kserve-ci-e2e-test resourceVersion: "20396" uid: 929e833d-d049-4d12-a994-03a8554da57d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj namespace: kserve-ci-e2e-test resourceVersion: "20325" uid: 69a9a267-5518-4d44-9461-fe6d2793a548 kind: Event lastTimestamp: "2026-04-25T00:15:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-25T00:15:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:15:36Z" name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj.18a97154a1826dcf namespace: kserve-ci-e2e-test resourceVersion: "20397" uid: c904be5a-4c85-4929-8b66-a2e499e719b1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-25T00:15:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj namespace: kserve-ci-e2e-test resourceVersion: "20325" uid: 69a9a267-5518-4d44-9461-fe6d2793a548 kind: Event lastTimestamp: "2026-04-25T00:16:53Z" message: 'Readiness probe failed: dial tcp 10.134.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-25T00:15:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:16:53Z" name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj.18a97154d5b9201d namespace: kserve-ci-e2e-test resourceVersion: "21025" uid: ac7acf2f-ad08-4203-aa20-1193198eac4e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:16:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj namespace: kserve-ci-e2e-test resourceVersion: "20325" uid: 69a9a267-5518-4d44-9461-fe6d2793a548 kind: Event lastTimestamp: "2026-04-25T00:16:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-25T00:16:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:16:51Z" name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj.18a9716627cbfea5 namespace: kserve-ci-e2e-test resourceVersion: "20974" uid: 7e5990ff-1b64-4c2f-83a3-867117804a75 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:16:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj namespace: kserve-ci-e2e-test resourceVersion: "20325" uid: 69a9a267-5518-4d44-9461-fe6d2793a548 kind: Event lastTimestamp: "2026-04-25T00:16:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-25T00:16:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:16:51Z" name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj.18a9716627cc7a06 namespace: kserve-ci-e2e-test resourceVersion: "20976" uid: 6586546b-ea37-48e7-b534-70ce3ecb07e9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:16:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj namespace: kserve-ci-e2e-test resourceVersion: "20325" uid: 69a9a267-5518-4d44-9461-fe6d2793a548 kind: Event lastTimestamp: "2026-04-25T00:16:53Z" message: 'Readiness probe failed: Get "https://10.134.0.47:8643/healthz": dial tcp 10.134.0.47:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-25T00:16:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:16:53Z" name: raw-sklearn-926cf-predictor-6fc8899c49-49qgj.18a9716687dcc78b namespace: kserve-ci-e2e-test resourceVersion: "21024" uid: d3f40407-05f2-43f8-ad79-e9bfed2116af reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-926cf-predictor-6fc8899c49 namespace: kserve-ci-e2e-test resourceVersion: "20313" uid: 1f019c6f-3d1d-494a-a900-2393a8fcd3f8 kind: Event lastTimestamp: "2026-04-25T00:15:31Z" message: 'Created pod: raw-sklearn-926cf-predictor-6fc8899c49-49qgj' metadata: creationTimestamp: "2026-04-25T00:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:15:31Z" name: raw-sklearn-926cf-predictor-6fc8899c49.18a971537ed03eae namespace: kserve-ci-e2e-test resourceVersion: "20324" uid: deec5f23-1c85-485c-8e62-c314e9ae9b3e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-926cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "20311" uid: 5b0d9296-2b38-4ed9-bf0c-1709ba560cbb kind: Event lastTimestamp: "2026-04-25T00:15:31Z" message: Scaled up replica set raw-sklearn-926cf-predictor-6fc8899c49 from 0 to 1 metadata: creationTimestamp: "2026-04-25T00:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:15:31Z" name: raw-sklearn-926cf-predictor.18a971537c67a701 namespace: kserve-ci-e2e-test resourceVersion: "20315" uid: 594e97a3-1644-4b03-a13e-d32f9ca017bd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-25T00:15:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-926cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "20312" uid: e4d1a11c-2acd-4b5f-94a9-04cf042711b0 kind: Event lastTimestamp: "2026-04-25T00:16:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-25T00:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:16:01Z" name: raw-sklearn-926cf-predictor.18a97156fb385158 namespace: kserve-ci-e2e-test resourceVersion: "20578" uid: 7f832f09-562d-43d6-a07c-71b51ded508b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-25T00:15:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-926cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "20312" uid: e4d1a11c-2acd-4b5f-94a9-04cf042711b0 kind: Event lastTimestamp: "2026-04-25T00:16:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-25T00:15:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:16:01Z" name: raw-sklearn-926cf-predictor.18a97156fb9d9408 namespace: kserve-ci-e2e-test resourceVersion: "20579" uid: c7e4b063-ab72-43c3-b06c-c80d6f563204 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-25T00:16:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-926cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "20476" uid: e4d1a11c-2acd-4b5f-94a9-04cf042711b0 kind: Event lastTimestamp: "2026-04-25T00:16:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-25T00:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:16:46Z" name: raw-sklearn-926cf-predictor.18a9715df95b9111 namespace: kserve-ci-e2e-test resourceVersion: "20910" uid: 1a614cd2-120c-4654-98ca-798ec7eb7206 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-25T00:16:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-926cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "20476" uid: e4d1a11c-2acd-4b5f-94a9-04cf042711b0 kind: Event lastTimestamp: "2026-04-25T00:16:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-25T00:16:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:16:46Z" name: raw-sklearn-926cf-predictor.18a9715df9c5f633 namespace: kserve-ci-e2e-test resourceVersion: "20911" uid: 16fbcc08-a763-4c13-922c-e580efcbb0df reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-25T00:15:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-926cf namespace: kserve-ci-e2e-test resourceVersion: "20301" uid: cc1de9d7-64df-47e4-b436-a6a5985bfb95 kind: Event lastTimestamp: "2026-04-25T00:16:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-25T00:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:16:43Z" name: raw-sklearn-926cf.18a971537c3fb6f3 namespace: kserve-ci-e2e-test resourceVersion: "20885" uid: 6bec3a22-31bd-4589-85e1-a808bed2fa03 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-25T00:15:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-926cf namespace: kserve-ci-e2e-test resourceVersion: "20301" uid: cc1de9d7-64df-47e4-b436-a6a5985bfb95 kind: Event lastTimestamp: "2026-04-25T00:15:31Z" message: 'Failed to update status for InferenceService "raw-sklearn-926cf": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-926cf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-25T00:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:15:31Z" name: raw-sklearn-926cf.18a971537cbdb9bd namespace: kserve-ci-e2e-test resourceVersion: "20338" uid: 7948d693-7139-4128-a34b-6c2d05caf86a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:15:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-926cf namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: cc1de9d7-64df-47e4-b436-a6a5985bfb95 kind: Event lastTimestamp: "2026-04-25T00:15:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-926cf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-25T00:15:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:15:31Z" name: raw-sklearn-926cf.18a971538325485a namespace: kserve-ci-e2e-test resourceVersion: "20339" uid: dfac1c7c-ac1c-455d-b4d8-21627d4704f9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:16:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-926cf namespace: kserve-ci-e2e-test resourceVersion: "20881" uid: cc1de9d7-64df-47e4-b436-a6a5985bfb95 kind: Event lastTimestamp: "2026-04-25T00:16:43Z" message: InferenceService [raw-sklearn-926cf] is Ready metadata: creationTimestamp: "2026-04-25T00:16:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:16:43Z" name: raw-sklearn-926cf.18a9716438bc7b2a namespace: kserve-ci-e2e-test resourceVersion: "20883" uid: 5842f721-620b-4a36-a867-d0022b11c292 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-25T00:16:51.617248Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d namespace: kserve-ci-e2e-test resourceVersion: "20989" uid: ce258e6b-bad3-41a8-81c4-90a9a3d84b5b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d to ip-10-0-132-64.ec2.internal metadata: creationTimestamp: "2026-04-25T00:16:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-25T00:16:51Z" name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d.18a971662c3e4e44 namespace: kserve-ci-e2e-test resourceVersion: "20995" uid: 9a680362-c52a-40d3-a8b7-8338117e6db3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56c7745b65-967mt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:16:52Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d namespace: kserve-ci-e2e-test resourceVersion: "20999" uid: ce258e6b-bad3-41a8-81c4-90a9a3d84b5b kind: Event lastTimestamp: "2026-04-25T00:16:52Z" message: Add eth0 [10.134.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-25T00:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-25T00:16:52Z" name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d.18a9716645a3e12e namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: 1924c698-f54d-478f-bd68-46a7a1f5bb66 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d namespace: kserve-ci-e2e-test resourceVersion: "20993" uid: ce258e6b-bad3-41a8-81c4-90a9a3d84b5b kind: Event lastTimestamp: "2026-04-25T00:16:52Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:72e9a53b4f6609f050c864ccf8bba3bc53b15bbbb0a09e5de74a3bdc7465dd84" already present on machine metadata: creationTimestamp: "2026-04-25T00:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:16:52Z" name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d.18a9716646fbca25 namespace: kserve-ci-e2e-test resourceVersion: "21006" uid: 4d660d46-9326-47a7-ae7c-b6c90d38cdff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d namespace: kserve-ci-e2e-test resourceVersion: "20993" uid: ce258e6b-bad3-41a8-81c4-90a9a3d84b5b kind: Event lastTimestamp: "2026-04-25T00:16:52Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-25T00:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:16:52Z" name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d.18a9716649b8b7ad namespace: kserve-ci-e2e-test resourceVersion: "21008" uid: b17b9c06-4a51-44ee-8512-e08a374838db reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d namespace: kserve-ci-e2e-test resourceVersion: "20993" uid: ce258e6b-bad3-41a8-81c4-90a9a3d84b5b kind: Event lastTimestamp: "2026-04-25T00:16:52Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-25T00:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:16:52Z" name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d.18a971664a2a936d namespace: kserve-ci-e2e-test resourceVersion: "21009" uid: 47208a37-7941-48ce-911a-e02b0a709768 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:16:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d namespace: kserve-ci-e2e-test resourceVersion: "20993" uid: ce258e6b-bad3-41a8-81c4-90a9a3d84b5b kind: Event lastTimestamp: "2026-04-25T00:16:56Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1435" already present on machine metadata: creationTimestamp: "2026-04-25T00:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:16:56Z" name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d.18a971674bab8527 namespace: kserve-ci-e2e-test resourceVersion: "21047" uid: e00c2e18-15ed-478d-a298-147ce03671f5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:16:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d namespace: kserve-ci-e2e-test resourceVersion: "20993" uid: ce258e6b-bad3-41a8-81c4-90a9a3d84b5b kind: Event lastTimestamp: "2026-04-25T00:16:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-25T00:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:16:56Z" name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d.18a971674f3d0a5e namespace: kserve-ci-e2e-test resourceVersion: "21053" uid: 8e311d7c-f293-4d9c-bf39-ee32b0b1cb66 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:16:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d namespace: kserve-ci-e2e-test resourceVersion: "20993" uid: ce258e6b-bad3-41a8-81c4-90a9a3d84b5b kind: Event lastTimestamp: "2026-04-25T00:16:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-25T00:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:16:56Z" name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d.18a971674fb56890 namespace: kserve-ci-e2e-test resourceVersion: "21054" uid: 7478ca5e-9991-49e0-8af8-4170ac777960 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:16:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d namespace: kserve-ci-e2e-test resourceVersion: "20993" uid: ce258e6b-bad3-41a8-81c4-90a9a3d84b5b kind: Event lastTimestamp: "2026-04-25T00:16:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-25T00:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:16:56Z" name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d.18a971674fc10052 namespace: kserve-ci-e2e-test resourceVersion: "21055" uid: fb99b38a-8427-4163-9317-aff5f5e1595c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:16:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d namespace: kserve-ci-e2e-test resourceVersion: "20993" uid: ce258e6b-bad3-41a8-81c4-90a9a3d84b5b kind: Event lastTimestamp: "2026-04-25T00:16:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-25T00:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:16:56Z" name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d.18a971675285c48e namespace: kserve-ci-e2e-test resourceVersion: "21057" uid: 8e7f0d42-d58f-44cb-91bb-ccd1b2d0dd00 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:16:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d namespace: kserve-ci-e2e-test resourceVersion: "20993" uid: ce258e6b-bad3-41a8-81c4-90a9a3d84b5b kind: Event lastTimestamp: "2026-04-25T00:16:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-25T00:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:16:56Z" name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d.18a9716752fdbbd0 namespace: kserve-ci-e2e-test resourceVersion: "21058" uid: fa8278f4-09d2-4de7-af7c-8b852ea5cc65 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-25T00:16:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d namespace: kserve-ci-e2e-test resourceVersion: "20993" uid: ce258e6b-bad3-41a8-81c4-90a9a3d84b5b kind: Event lastTimestamp: "2026-04-25T00:18:13Z" message: 'Readiness probe failed: dial tcp 10.134.0.48:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-25T00:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:18:13Z" name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d.18a9716787a7df93 namespace: kserve-ci-e2e-test resourceVersion: "21668" uid: 26da8116-c787-4f39-ad55-ab8ed978b8c9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:18:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d namespace: kserve-ci-e2e-test resourceVersion: "20993" uid: ce258e6b-bad3-41a8-81c4-90a9a3d84b5b kind: Event lastTimestamp: "2026-04-25T00:18:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-25T00:18:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:18:11Z" name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d.18a97178d30e7084 namespace: kserve-ci-e2e-test resourceVersion: "21621" uid: 83d439d8-ea8d-4e29-a415-558a5839d4c7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:18:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d namespace: kserve-ci-e2e-test resourceVersion: "20993" uid: ce258e6b-bad3-41a8-81c4-90a9a3d84b5b kind: Event lastTimestamp: "2026-04-25T00:18:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-25T00:18:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:18:11Z" name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d.18a97178d30ef79b namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: 0eccd610-b82b-49ac-8fc5-d710a8941aa7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:18:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d namespace: kserve-ci-e2e-test resourceVersion: "20993" uid: ce258e6b-bad3-41a8-81c4-90a9a3d84b5b kind: Event lastTimestamp: "2026-04-25T00:18:13Z" message: 'Readiness probe failed: Get "https://10.134.0.48:8643/healthz": dial tcp 10.134.0.48:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-25T00:18:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-25T00:18:13Z" name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d.18a9717939e3f215 namespace: kserve-ci-e2e-test resourceVersion: "21667" uid: 9c0c13c0-ffcd-4276-9ddc-6e233960bf8b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-64.ec2.internal source: component: kubelet host: ip-10-0-132-64.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:16:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5 namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: c60b6b38-e7da-4e48-bed7-0f76c04e528d kind: Event lastTimestamp: "2026-04-25T00:16:51Z" message: 'Created pod: raw-sklearn-runtime-bca52-predictor-7cc55b85d5-nl97d' metadata: creationTimestamp: "2026-04-25T00:16:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:16:51Z" name: raw-sklearn-runtime-bca52-predictor-7cc55b85d5.18a971662bb87d39 namespace: kserve-ci-e2e-test resourceVersion: "20992" uid: 77cf9da2-698e-4feb-bc49-06fa4551e6e1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:16:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-bca52-predictor namespace: kserve-ci-e2e-test resourceVersion: "20975" uid: 0a87b7b8-f3ae-473c-88a5-3333d7121068 kind: Event lastTimestamp: "2026-04-25T00:16:51Z" message: Scaled up replica set raw-sklearn-runtime-bca52-predictor-7cc55b85d5 from 0 to 1 metadata: creationTimestamp: "2026-04-25T00:16:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:16:51Z" name: raw-sklearn-runtime-bca52-predictor.18a9716628e54264 namespace: kserve-ci-e2e-test resourceVersion: "20980" uid: 0211bd6a-b9a4-4438-84e6-79fa55b103b5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-25T00:17:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-bca52-predictor namespace: kserve-ci-e2e-test resourceVersion: "20977" uid: c8ea036e-332b-4d4e-9f96-830e8c86958e kind: Event lastTimestamp: "2026-04-25T00:17:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-25T00:17:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:17:36Z" name: raw-sklearn-runtime-bca52-predictor.18a97169a75f1d5e namespace: kserve-ci-e2e-test resourceVersion: "21348" uid: 960e8e97-a6b5-48be-a66f-80e8c1106ecd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-25T00:17:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-bca52-predictor namespace: kserve-ci-e2e-test resourceVersion: "20977" uid: c8ea036e-332b-4d4e-9f96-830e8c86958e kind: Event lastTimestamp: "2026-04-25T00:17:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-25T00:17:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:17:36Z" name: raw-sklearn-runtime-bca52-predictor.18a97169a7c360f7 namespace: kserve-ci-e2e-test resourceVersion: "21349" uid: 8e4d47be-cd92-45d1-b7a9-1c013858a922 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-25T00:17:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-bca52-predictor namespace: kserve-ci-e2e-test resourceVersion: "21135" uid: c8ea036e-332b-4d4e-9f96-830e8c86958e kind: Event lastTimestamp: "2026-04-25T00:18:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-25T00:17:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:18:06Z" name: raw-sklearn-runtime-bca52-predictor.18a9716d26bdc487 namespace: kserve-ci-e2e-test resourceVersion: "21568" uid: 47cc82fa-3ecf-4df1-955b-5e1360d76a52 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-25T00:17:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-bca52-predictor namespace: kserve-ci-e2e-test resourceVersion: "21135" uid: c8ea036e-332b-4d4e-9f96-830e8c86958e kind: Event lastTimestamp: "2026-04-25T00:18:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-25T00:17:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-25T00:18:06Z" name: raw-sklearn-runtime-bca52-predictor.18a9716d271ec6e0 namespace: kserve-ci-e2e-test resourceVersion: "21569" uid: d53b8906-a9a9-4f64-aac5-537e48a9082c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-25T00:16:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-bca52 namespace: kserve-ci-e2e-test resourceVersion: "20947" uid: d3255d95-65ef-4710-a296-d9719928bb10 kind: Event lastTimestamp: "2026-04-25T00:18:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-25T00:16:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:18:03Z" name: raw-sklearn-runtime-bca52.18a9716628bdba6f namespace: kserve-ci-e2e-test resourceVersion: "21551" uid: 0587b788-5221-4d8c-b856-71526c7e3295 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:16:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-bca52 namespace: kserve-ci-e2e-test resourceVersion: "20947" uid: d3255d95-65ef-4710-a296-d9719928bb10 kind: Event lastTimestamp: "2026-04-25T00:16:51Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-bca52": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-bca52": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-25T00:16:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:16:51Z" name: raw-sklearn-runtime-bca52.18a971662910eabc namespace: kserve-ci-e2e-test resourceVersion: "20982" uid: 9fbd1388-fec4-4492-95d7-786b43de2a95 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-25T00:18:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-bca52 namespace: kserve-ci-e2e-test resourceVersion: "21549" uid: d3255d95-65ef-4710-a296-d9719928bb10 kind: Event lastTimestamp: "2026-04-25T00:18:03Z" message: InferenceService [raw-sklearn-runtime-bca52] is Ready metadata: creationTimestamp: "2026-04-25T00:18:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-25T00:18:03Z" name: raw-sklearn-runtime-bca52.18a97176eac8ce9d namespace: kserve-ci-e2e-test resourceVersion: "21550" uid: bce5a1b6-8783-4ce6-a5cf-80904daac51f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "21823"