--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:21:20.443537Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-e3c30a-predictor-578df99bf4-drcxv namespace: kserve-ci-e2e-test resourceVersion: "20669" uid: 55984685-7f0b-473b-a35d-20ad505217a6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-e3c30a-predictor-578df99bf4-drcxv to ip-10-0-134-26.ec2.internal metadata: creationTimestamp: "2026-07-10T19:21:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:21:20Z" name: isvc-init-fail-e3c30a-predictor-578df99bf4-drcxv.18c103f584b1e748 namespace: kserve-ci-e2e-test resourceVersion: "20675" uid: e12a0ffa-3dbd-4b06-86cf-09954792dd6a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c9dd4b6f4-9plfp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:20Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-e3c30a-predictor-578df99bf4-drcxv namespace: kserve-ci-e2e-test resourceVersion: "20679" uid: 55984685-7f0b-473b-a35d-20ad505217a6 kind: Event lastTimestamp: "2026-07-10T19:21:20Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:21:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:21:20Z" name: isvc-init-fail-e3c30a-predictor-578df99bf4-drcxv.18c103f59df5b438 namespace: kserve-ci-e2e-test resourceVersion: "20688" uid: 6fcb5447-2c86-4b2c-aaa5-664e73b9a9c1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:21:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-e3c30a-predictor-578df99bf4-drcxv namespace: kserve-ci-e2e-test resourceVersion: "20671" uid: 55984685-7f0b-473b-a35d-20ad505217a6 kind: Event lastTimestamp: "2026-07-10T19:21:25Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7893a590791cc9f33e3a3207dcc1d0843778bbf1aee8b6f22a63e3aa3b91a2e1" already present on machine metadata: creationTimestamp: "2026-07-10T19:21:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:25Z" name: isvc-init-fail-e3c30a-predictor-578df99bf4-drcxv.18c103f59f529f30 namespace: kserve-ci-e2e-test resourceVersion: "20733" uid: ec88d96a-3af7-44e4-aa86-7ee0b0014d0d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:21:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-e3c30a-predictor-578df99bf4-drcxv namespace: kserve-ci-e2e-test resourceVersion: "20671" uid: 55984685-7f0b-473b-a35d-20ad505217a6 kind: Event lastTimestamp: "2026-07-10T19:21:25Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:21:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:25Z" name: isvc-init-fail-e3c30a-predictor-578df99bf4-drcxv.18c103f5a23cdc13 namespace: kserve-ci-e2e-test resourceVersion: "20736" uid: 3eb33287-622d-479a-9feb-f77984e5aca5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:21:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-e3c30a-predictor-578df99bf4-drcxv namespace: kserve-ci-e2e-test resourceVersion: "20671" uid: 55984685-7f0b-473b-a35d-20ad505217a6 kind: Event lastTimestamp: "2026-07-10T19:21:25Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:21:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:25Z" name: isvc-init-fail-e3c30a-predictor-578df99bf4-drcxv.18c103f5a2aae1b7 namespace: kserve-ci-e2e-test resourceVersion: "20737" uid: e7e6c487-4956-4e05-bace-43a4fda671b1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-e3c30a-predictor-578df99bf4-drcxv namespace: kserve-ci-e2e-test resourceVersion: "20671" uid: 55984685-7f0b-473b-a35d-20ad505217a6 kind: Event lastTimestamp: "2026-07-10T19:21:30Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-07-10T19:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:30Z" name: isvc-init-fail-e3c30a-predictor-578df99bf4-drcxv.18c103f7d9231fa7 namespace: kserve-ci-e2e-test resourceVersion: "20789" uid: a0b06d92-da81-43f9-b54d-64e410d85edc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-e3c30a-predictor-578df99bf4 namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: 6cb5b9dc-9960-47f2-bf08-bd5a40194624 kind: Event lastTimestamp: "2026-07-10T19:21:20Z" message: 'Created pod: isvc-init-fail-e3c30a-predictor-578df99bf4-drcxv' metadata: creationTimestamp: "2026-07-10T19:21:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:20Z" name: isvc-init-fail-e3c30a-predictor-578df99bf4.18c103f5843f7fec namespace: kserve-ci-e2e-test resourceVersion: "20670" uid: c5c5d597-2310-4951-a0bd-2005171425b3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-e3c30a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20660" uid: 6798ea6d-5b94-4306-81a7-af7da7c6f82d kind: Event lastTimestamp: "2026-07-10T19:21:20Z" message: Scaled up replica set isvc-init-fail-e3c30a-predictor-578df99bf4 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:21:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:20Z" name: isvc-init-fail-e3c30a-predictor.18c103f58139f612 namespace: kserve-ci-e2e-test resourceVersion: "20663" uid: 2cca3732-e638-4826-ae23-55e082e77c83 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-10T19:21:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-e3c30a namespace: kserve-ci-e2e-test resourceVersion: "20651" uid: 5b384b06-04ab-4084-ad50-28ec992ab3f8 kind: Event lastTimestamp: "2026-07-10T19:21:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:21:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:21:26Z" name: isvc-init-fail-e3c30a.18c103f5813a8275 namespace: kserve-ci-e2e-test resourceVersion: "20740" uid: 486bbfbe-bd3d-40ba-a719-2f904d4b676a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-e3c30a namespace: kserve-ci-e2e-test resourceVersion: "20651" uid: 5b384b06-04ab-4084-ad50-28ec992ab3f8 kind: Event lastTimestamp: "2026-07-10T19:21:20Z" message: 'Failed to update status for InferenceService "isvc-init-fail-e3c30a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-e3c30a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:21:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:21:20Z" name: isvc-init-fail-e3c30a.18c103f581a47a15 namespace: kserve-ci-e2e-test resourceVersion: "20666" uid: 8bcca815-dabd-471d-bb0d-640a45ccd296 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-e3c30a namespace: kserve-ci-e2e-test resourceVersion: "20651" uid: 5b384b06-04ab-4084-ad50-28ec992ab3f8 kind: Event lastTimestamp: "2026-07-10T19:21:20Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-e3c30a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:21:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:21:20Z" name: isvc-init-fail-e3c30a.18c103f581a4b5af namespace: kserve-ci-e2e-test resourceVersion: "20667" uid: 2d7508ae-2858-4b31-830a-177add7bdf00 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:15:06.322817Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn namespace: kserve-ci-e2e-test resourceVersion: "17628" uid: 2d9d6054-06e5-4f7f-abfe-c30d335fae19 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn to ip-10-0-134-26.ec2.internal metadata: creationTimestamp: "2026-07-10T19:15:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:15:06Z" name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn.18c1039e695cc2e6 namespace: kserve-ci-e2e-test resourceVersion: "17633" uid: 482744f2-f8fe-442d-832a-d3e0e62e4b99 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c9dd4b6f4-9plfp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:06Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn namespace: kserve-ci-e2e-test resourceVersion: "17637" uid: 2d9d6054-06e5-4f7f-abfe-c30d335fae19 kind: Event lastTimestamp: "2026-07-10T19:15:06Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:15:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:15:06Z" name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn.18c1039e83202c27 namespace: kserve-ci-e2e-test resourceVersion: "17644" uid: 09d32b6e-0ca2-4680-a607-9d9afa919c43 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn namespace: kserve-ci-e2e-test resourceVersion: "17631" uid: 2d9d6054-06e5-4f7f-abfe-c30d335fae19 kind: Event lastTimestamp: "2026-07-10T19:15:06Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7893a590791cc9f33e3a3207dcc1d0843778bbf1aee8b6f22a63e3aa3b91a2e1" already present on machine metadata: creationTimestamp: "2026-07-10T19:15:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:15:06Z" name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn.18c1039e8497329c namespace: kserve-ci-e2e-test resourceVersion: "17647" uid: c05f6192-a948-4dcb-b427-e74068441662 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn namespace: kserve-ci-e2e-test resourceVersion: "17631" uid: 2d9d6054-06e5-4f7f-abfe-c30d335fae19 kind: Event lastTimestamp: "2026-07-10T19:15:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:15:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:15:06Z" name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn.18c1039e877e0678 namespace: kserve-ci-e2e-test resourceVersion: "17648" uid: 077f7c48-09ce-46c1-9a63-eeec9780528f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn namespace: kserve-ci-e2e-test resourceVersion: "17631" uid: 2d9d6054-06e5-4f7f-abfe-c30d335fae19 kind: Event lastTimestamp: "2026-07-10T19:15:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:15:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:15:06Z" name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn.18c1039e87f1f622 namespace: kserve-ci-e2e-test resourceVersion: "17649" uid: 0a8e94d6-2a1b-42dd-ae0b-6deb6015fa4a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn namespace: kserve-ci-e2e-test resourceVersion: "17631" uid: 2d9d6054-06e5-4f7f-abfe-c30d335fae19 kind: Event lastTimestamp: "2026-07-10T19:15:11Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:15:11Z" name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn.18c1039f97fcfb2d namespace: kserve-ci-e2e-test resourceVersion: "17690" uid: e2fa285d-07c9-454a-9a50-0d8c82370c01 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn namespace: kserve-ci-e2e-test resourceVersion: "17631" uid: 2d9d6054-06e5-4f7f-abfe-c30d335fae19 kind: Event lastTimestamp: "2026-07-10T19:15:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:15:11Z" name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn.18c1039f9ae62e1f namespace: kserve-ci-e2e-test resourceVersion: "17694" uid: 2d54e281-cab4-4011-abaf-4a5fb95e3d3f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn namespace: kserve-ci-e2e-test resourceVersion: "17631" uid: 2d9d6054-06e5-4f7f-abfe-c30d335fae19 kind: Event lastTimestamp: "2026-07-10T19:15:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:15:11Z" name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn.18c1039f9b5a78ff namespace: kserve-ci-e2e-test resourceVersion: "17695" uid: 308bf7e7-b252-417b-87dc-a65fa05a2ec8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn namespace: kserve-ci-e2e-test resourceVersion: "17631" uid: 2d9d6054-06e5-4f7f-abfe-c30d335fae19 kind: Event lastTimestamp: "2026-07-10T19:15:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:15:11Z" name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn.18c1039f9b64eb77 namespace: kserve-ci-e2e-test resourceVersion: "17697" uid: 1490c5d9-2899-4886-bef4-e8dda77cca43 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn namespace: kserve-ci-e2e-test resourceVersion: "17631" uid: 2d9d6054-06e5-4f7f-abfe-c30d335fae19 kind: Event lastTimestamp: "2026-07-10T19:15:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:15:11Z" name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn.18c1039f9f14b89d namespace: kserve-ci-e2e-test resourceVersion: "17701" uid: 7195eb70-0516-4b88-9f48-50b97884f6d5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn namespace: kserve-ci-e2e-test resourceVersion: "17631" uid: 2d9d6054-06e5-4f7f-abfe-c30d335fae19 kind: Event lastTimestamp: "2026-07-10T19:15:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:15:11Z" name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn.18c1039f9f87dba2 namespace: kserve-ci-e2e-test resourceVersion: "17702" uid: b1f20459-1ae4-4eec-a657-b20cb336f2c7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn namespace: kserve-ci-e2e-test resourceVersion: "17631" uid: 2d9d6054-06e5-4f7f-abfe-c30d335fae19 kind: Event lastTimestamp: "2026-07-10T19:15:11Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:4c6a130c213136d803ee987407e3094d900670800a73507fd61c7fd7f0573052" already present on machine metadata: creationTimestamp: "2026-07-10T19:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:15:11Z" name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn.18c1039f9f92422b namespace: kserve-ci-e2e-test resourceVersion: "17703" uid: 5fd5ad3f-b82d-47ac-9bf3-9271ad4e5c46 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn namespace: kserve-ci-e2e-test resourceVersion: "17631" uid: 2d9d6054-06e5-4f7f-abfe-c30d335fae19 kind: Event lastTimestamp: "2026-07-10T19:15:11Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-10T19:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:15:11Z" name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn.18c1039fa2b72381 namespace: kserve-ci-e2e-test resourceVersion: "17704" uid: 65f2f1e8-443e-4b32-a9fe-4a3a4bad415a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn namespace: kserve-ci-e2e-test resourceVersion: "17631" uid: 2d9d6054-06e5-4f7f-abfe-c30d335fae19 kind: Event lastTimestamp: "2026-07-10T19:15:11Z" message: Started container agent metadata: creationTimestamp: "2026-07-10T19:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:15:11Z" name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn.18c1039fa32ce11f namespace: kserve-ci-e2e-test resourceVersion: "17705" uid: 69a7610a-dd77-4037-9014-5c8016c1ca7b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-10T19:15:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn namespace: kserve-ci-e2e-test resourceVersion: "17631" uid: 2d9d6054-06e5-4f7f-abfe-c30d335fae19 kind: Event lastTimestamp: "2026-07-10T19:16:58Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:15:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:16:58Z" name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn.18c1039fd4128f34 namespace: kserve-ci-e2e-test resourceVersion: "18600" uid: 6e8a19c0-7b18-4272-8bf8-1703d8f3048b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-10T19:15:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn namespace: kserve-ci-e2e-test resourceVersion: "17631" uid: 2d9d6054-06e5-4f7f-abfe-c30d335fae19 kind: Event lastTimestamp: "2026-07-10T19:16:48Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-10T19:15:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:16:48Z" name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn.18c1039fd41d17bb namespace: kserve-ci-e2e-test resourceVersion: "18525" uid: 5793de2f-d41c-44b4-a114-b4e976a00bfd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:16:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn namespace: kserve-ci-e2e-test resourceVersion: "17631" uid: 2d9d6054-06e5-4f7f-abfe-c30d335fae19 kind: Event lastTimestamp: "2026-07-10T19:16:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:16:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:16:41Z" name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn.18c103b4968f9e8e namespace: kserve-ci-e2e-test resourceVersion: "18401" uid: bd022348-4765-4514-ab9d-85c8c7447571 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:16:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn namespace: kserve-ci-e2e-test resourceVersion: "17631" uid: 2d9d6054-06e5-4f7f-abfe-c30d335fae19 kind: Event lastTimestamp: "2026-07-10T19:16:41Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-10T19:16:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:16:41Z" name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn.18c103b4968fddf0 namespace: kserve-ci-e2e-test resourceVersion: "18403" uid: e01c5627-f533-41e9-8980-a458f296ed0a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:16:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn namespace: kserve-ci-e2e-test resourceVersion: "17631" uid: 2d9d6054-06e5-4f7f-abfe-c30d335fae19 kind: Event lastTimestamp: "2026-07-10T19:16:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:16:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:16:41Z" name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn.18c103b496900b70 namespace: kserve-ci-e2e-test resourceVersion: "18407" uid: 48c85cec-d2a3-4ad7-97a7-89b766a4a0b9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:16:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn namespace: kserve-ci-e2e-test resourceVersion: "17631" uid: 2d9d6054-06e5-4f7f-abfe-c30d335fae19 kind: Event lastTimestamp: "2026-07-10T19:16:58Z" message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp 10.133.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:16:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:16:58Z" name: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn.18c103b5046865f8 namespace: kserve-ci-e2e-test resourceVersion: "18599" uid: 097e5388-5d9d-4750-8f99-55e17283409e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-358e7-predictor-74b9cfb87c namespace: kserve-ci-e2e-test resourceVersion: "17617" uid: 508b6e46-4a1f-466f-9801-d5805f6e9026 kind: Event lastTimestamp: "2026-07-10T19:15:06Z" message: 'Created pod: isvc-logger-raw-358e7-predictor-74b9cfb87c-sn8qn' metadata: creationTimestamp: "2026-07-10T19:15:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:15:06Z" name: isvc-logger-raw-358e7-predictor-74b9cfb87c.18c1039e68aef8e3 namespace: kserve-ci-e2e-test resourceVersion: "17630" uid: 84964702-0d19-4fa2-a03b-e47534cddfcb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-358e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17615" uid: c9b84d2d-2744-43ba-9199-a6325f26b2e2 kind: Event lastTimestamp: "2026-07-10T19:15:06Z" message: Scaled up replica set isvc-logger-raw-358e7-predictor-74b9cfb87c from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:15:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:15:06Z" name: isvc-logger-raw-358e7-predictor.18c1039e6508bf33 namespace: kserve-ci-e2e-test resourceVersion: "17618" uid: 0952bf42-f36b-4f71-8ecf-8a69e818473d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:15:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-358e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17616" uid: 3fc0fb65-b789-4734-8a92-577abd5eb058 kind: Event lastTimestamp: "2026-07-10T19:15:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:15:36Z" name: isvc-logger-raw-358e7-predictor.18c103a1e40c1e1a namespace: kserve-ci-e2e-test resourceVersion: "17895" uid: 3252c8b2-64f5-4658-a638-98e4f354a610 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:15:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-358e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17616" uid: 3fc0fb65-b789-4734-8a92-577abd5eb058 kind: Event lastTimestamp: "2026-07-10T19:15:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:15:36Z" name: isvc-logger-raw-358e7-predictor.18c103a1e497a672 namespace: kserve-ci-e2e-test resourceVersion: "17896" uid: 0ce44d91-63c5-46c0-87c0-de86b2584d89 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:15:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-358e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17781" uid: 3fc0fb65-b789-4734-8a92-577abd5eb058 kind: Event lastTimestamp: "2026-07-10T19:16:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:15:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:16:36Z" name: isvc-logger-raw-358e7-predictor.18c103a8e2566ae8 namespace: kserve-ci-e2e-test resourceVersion: "18331" uid: 14fa1ba5-108d-499c-99ab-a9c0abc4afca reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:15:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-358e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17781" uid: 3fc0fb65-b789-4734-8a92-577abd5eb058 kind: Event lastTimestamp: "2026-07-10T19:16:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:15:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:16:36Z" name: isvc-logger-raw-358e7-predictor.18c103a8e2b85042 namespace: kserve-ci-e2e-test resourceVersion: "18332" uid: f6cad3b6-b9ac-4a91-ba6f-e8962c61e0d4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-10T19:15:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-358e7 namespace: kserve-ci-e2e-test resourceVersion: "17608" uid: c8ce014c-85b6-477a-971f-fe0a970ff532 kind: Event lastTimestamp: "2026-07-10T19:16:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:15:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:16:28Z" name: isvc-logger-raw-358e7.18c1039e65185ad4 namespace: kserve-ci-e2e-test resourceVersion: "18277" uid: f587f508-1fd8-4a28-8e7c-4bd6584f4337 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-358e7 namespace: kserve-ci-e2e-test resourceVersion: "17608" uid: c8ce014c-85b6-477a-971f-fe0a970ff532 kind: Event lastTimestamp: "2026-07-10T19:15:06Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-358e7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-358e7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:15:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:15:06Z" name: isvc-logger-raw-358e7.18c1039e65a99254 namespace: kserve-ci-e2e-test resourceVersion: "17622" uid: cae5d15b-ed0b-48fd-82ec-c15c0df360ec reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:16:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-358e7 namespace: kserve-ci-e2e-test resourceVersion: "18275" uid: c8ce014c-85b6-477a-971f-fe0a970ff532 kind: Event lastTimestamp: "2026-07-10T19:16:28Z" message: InferenceService [isvc-logger-raw-358e7] is Ready metadata: creationTimestamp: "2026-07-10T19:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:16:28Z" name: isvc-logger-raw-358e7.18c103b18da275a2 namespace: kserve-ci-e2e-test resourceVersion: "18276" uid: 9161d00d-1360-4d43-b9f6-22a7fe8e0b3e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:19:27.000351Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-147ca1-predictor-86b99c977c-956fs namespace: kserve-ci-e2e-test resourceVersion: "19692" uid: 340d0984-a7ef-4996-99cb-cc7bf0aa4af9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-147ca1-predictor-86b99c977c-956fs to ip-10-0-134-26.ec2.internal metadata: creationTimestamp: "2026-07-10T19:19:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:19:27Z" name: isvc-primary-147ca1-predictor-86b99c977c-956fs.18c103db1af435c8 namespace: kserve-ci-e2e-test resourceVersion: "19696" uid: 9a63562e-b8ec-4e64-8336-7564a6ef07ec reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c9dd4b6f4-9plfp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:27Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-147ca1-predictor-86b99c977c-956fs namespace: kserve-ci-e2e-test resourceVersion: "19703" uid: 340d0984-a7ef-4996-99cb-cc7bf0aa4af9 kind: Event lastTimestamp: "2026-07-10T19:19:27Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:19:27Z" name: isvc-primary-147ca1-predictor-86b99c977c-956fs.18c103db348d9ac0 namespace: kserve-ci-e2e-test resourceVersion: "19710" uid: 6b0dd16e-6a9f-4d7c-9e8f-7d4bd081c9cb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-147ca1-predictor-86b99c977c-956fs namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 340d0984-a7ef-4996-99cb-cc7bf0aa4af9 kind: Event lastTimestamp: "2026-07-10T19:19:27Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7893a590791cc9f33e3a3207dcc1d0843778bbf1aee8b6f22a63e3aa3b91a2e1" already present on machine metadata: creationTimestamp: "2026-07-10T19:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:27Z" name: isvc-primary-147ca1-predictor-86b99c977c-956fs.18c103db35ead179 namespace: kserve-ci-e2e-test resourceVersion: "19712" uid: c3d9c235-7378-436d-81f0-33a995cbcc6c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-147ca1-predictor-86b99c977c-956fs namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 340d0984-a7ef-4996-99cb-cc7bf0aa4af9 kind: Event lastTimestamp: "2026-07-10T19:19:27Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:27Z" name: isvc-primary-147ca1-predictor-86b99c977c-956fs.18c103db38cac61a namespace: kserve-ci-e2e-test resourceVersion: "19713" uid: d4e8e061-a522-4333-a73a-15c585d5a977 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-147ca1-predictor-86b99c977c-956fs namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 340d0984-a7ef-4996-99cb-cc7bf0aa4af9 kind: Event lastTimestamp: "2026-07-10T19:19:27Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:27Z" name: isvc-primary-147ca1-predictor-86b99c977c-956fs.18c103db39415132 namespace: kserve-ci-e2e-test resourceVersion: "19715" uid: 61d28974-ea34-4f4b-858e-9c5636267899 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-147ca1-predictor-86b99c977c-956fs namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 340d0984-a7ef-4996-99cb-cc7bf0aa4af9 kind: Event lastTimestamp: "2026-07-10T19:19:31Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:19:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:31Z" name: isvc-primary-147ca1-predictor-86b99c977c-956fs.18c103dc247b03e3 namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: 7e3f6ec9-9c63-438e-ab59-f08617dfe5f4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-147ca1-predictor-86b99c977c-956fs namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 340d0984-a7ef-4996-99cb-cc7bf0aa4af9 kind: Event lastTimestamp: "2026-07-10T19:19:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:19:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:31Z" name: isvc-primary-147ca1-predictor-86b99c977c-956fs.18c103dc280f1890 namespace: kserve-ci-e2e-test resourceVersion: "19754" uid: 882a2bab-c247-45ed-9ff5-bfd7c0b33f95 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-147ca1-predictor-86b99c977c-956fs namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 340d0984-a7ef-4996-99cb-cc7bf0aa4af9 kind: Event lastTimestamp: "2026-07-10T19:19:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:19:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:31Z" name: isvc-primary-147ca1-predictor-86b99c977c-956fs.18c103dc28a9a6fe namespace: kserve-ci-e2e-test resourceVersion: "19755" uid: 1ad20e8e-7cfe-4bfd-9f86-1a551b3b9160 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-147ca1-predictor-86b99c977c-956fs namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 340d0984-a7ef-4996-99cb-cc7bf0aa4af9 kind: Event lastTimestamp: "2026-07-10T19:19:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:19:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:31Z" name: isvc-primary-147ca1-predictor-86b99c977c-956fs.18c103dc28b3ea1d namespace: kserve-ci-e2e-test resourceVersion: "19756" uid: 3e7ee8e3-2d11-4edc-8a51-ffd6efa1c683 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-147ca1-predictor-86b99c977c-956fs namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 340d0984-a7ef-4996-99cb-cc7bf0aa4af9 kind: Event lastTimestamp: "2026-07-10T19:19:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:19:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:31Z" name: isvc-primary-147ca1-predictor-86b99c977c-956fs.18c103dc2ed0b13e namespace: kserve-ci-e2e-test resourceVersion: "19759" uid: 14e63d53-72a2-4a11-bb49-dad7b74af2ba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-147ca1-predictor-86b99c977c-956fs namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 340d0984-a7ef-4996-99cb-cc7bf0aa4af9 kind: Event lastTimestamp: "2026-07-10T19:19:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:19:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:31Z" name: isvc-primary-147ca1-predictor-86b99c977c-956fs.18c103dc2f49ff22 namespace: kserve-ci-e2e-test resourceVersion: "19760" uid: 5cf3e6e2-8c2b-427a-a50d-e37df30dd6ea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-10T19:19:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-147ca1-predictor-86b99c977c-956fs namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 340d0984-a7ef-4996-99cb-cc7bf0aa4af9 kind: Event lastTimestamp: "2026-07-10T19:21:19Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:19:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:19Z" name: isvc-primary-147ca1-predictor-86b99c977c-956fs.18c103dc608bb63e namespace: kserve-ci-e2e-test resourceVersion: "20636" uid: f3e72a21-438b-4d96-b180-3e0e4515c214 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-147ca1-predictor-86b99c977c-956fs namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 340d0984-a7ef-4996-99cb-cc7bf0aa4af9 kind: Event lastTimestamp: "2026-07-10T19:21:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:21:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:15Z" name: isvc-primary-147ca1-predictor-86b99c977c-956fs.18c103f451065dac namespace: kserve-ci-e2e-test resourceVersion: "20602" uid: 589201d0-51d5-40fc-8cd6-f6cb1bd1703f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-147ca1-predictor-86b99c977c-956fs namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 340d0984-a7ef-4996-99cb-cc7bf0aa4af9 kind: Event lastTimestamp: "2026-07-10T19:21:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:21:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:15Z" name: isvc-primary-147ca1-predictor-86b99c977c-956fs.18c103f45106ed7f namespace: kserve-ci-e2e-test resourceVersion: "20603" uid: e8e729d6-6f5f-4b56-b2f6-8abb7ac3b818 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-147ca1-predictor-86b99c977c-956fs namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 340d0984-a7ef-4996-99cb-cc7bf0aa4af9 kind: Event lastTimestamp: "2026-07-10T19:21:19Z" message: 'Readiness probe failed: Get "https://10.133.0.33:8643/healthz": dial tcp 10.133.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:21:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:19Z" name: isvc-primary-147ca1-predictor-86b99c977c-956fs.18c103f54ad10d49 namespace: kserve-ci-e2e-test resourceVersion: "20635" uid: a9428812-8051-4f5b-9554-9b0e1b2bcc20 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-147ca1-predictor-86b99c977c namespace: kserve-ci-e2e-test resourceVersion: "19683" uid: 53955923-0e33-421c-bb9f-b34a70675065 kind: Event lastTimestamp: "2026-07-10T19:19:26Z" message: 'Created pod: isvc-primary-147ca1-predictor-86b99c977c-956fs' metadata: creationTimestamp: "2026-07-10T19:19:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:19:26Z" name: isvc-primary-147ca1-predictor-86b99c977c.18c103db1a53f719 namespace: kserve-ci-e2e-test resourceVersion: "19693" uid: dbecfc28-ada7-4b7d-aa7c-e30cca2e1cf4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-147ca1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19680" uid: 4493ab1b-a01f-4ab1-af8d-b3338821d967 kind: Event lastTimestamp: "2026-07-10T19:19:26Z" message: Scaled up replica set isvc-primary-147ca1-predictor-86b99c977c from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:19:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:19:26Z" name: isvc-primary-147ca1-predictor.18c103db15a3aafb namespace: kserve-ci-e2e-test resourceVersion: "19684" uid: f18a9a3b-6e4c-437e-a76c-e6d6a0affcbc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:19:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-147ca1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19682" uid: c9acf44f-0777-4993-96d9-006b34e1c293 kind: Event lastTimestamp: "2026-07-10T19:20:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:11Z" name: isvc-primary-147ca1-predictor.18c103de944a4fe4 namespace: kserve-ci-e2e-test resourceVersion: "20046" uid: fe8fc961-25f0-45ba-b07c-408001598fe9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:19:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-147ca1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19682" uid: c9acf44f-0777-4993-96d9-006b34e1c293 kind: Event lastTimestamp: "2026-07-10T19:20:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:11Z" name: isvc-primary-147ca1-predictor.18c103de94bba755 namespace: kserve-ci-e2e-test resourceVersion: "20047" uid: 1d58091b-2032-4fef-bcf9-1c930c58383b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:20:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-147ca1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19839" uid: c9acf44f-0777-4993-96d9-006b34e1c293 kind: Event lastTimestamp: "2026-07-10T19:21:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:20:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:12Z" name: isvc-primary-147ca1-predictor.18c103e911cdd4fb namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: 52dc5abf-ce78-420d-8ca0-16e860f37309 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:20:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-147ca1-predictor namespace: kserve-ci-e2e-test resourceVersion: "19839" uid: c9acf44f-0777-4993-96d9-006b34e1c293 kind: Event lastTimestamp: "2026-07-10T19:21:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:20:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:12Z" name: isvc-primary-147ca1-predictor.18c103e912337c63 namespace: kserve-ci-e2e-test resourceVersion: "20543" uid: 7086b833-458c-4f32-819d-fa3fc39b2772 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-10T19:19:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-147ca1 namespace: kserve-ci-e2e-test resourceVersion: "19673" uid: b17420f4-e8a9-46e8-8b3a-898ed29bf980 kind: Event lastTimestamp: "2026-07-10T19:20:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:19:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:20:49Z" name: isvc-primary-147ca1.18c103db15d295a6 namespace: kserve-ci-e2e-test resourceVersion: "20319" uid: ea913873-8517-41cc-9e29-ac60c89ca296 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:19:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-147ca1 namespace: kserve-ci-e2e-test resourceVersion: "19673" uid: b17420f4-e8a9-46e8-8b3a-898ed29bf980 kind: Event lastTimestamp: "2026-07-10T19:19:27Z" message: 'Failed to update status for InferenceService "isvc-primary-147ca1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-147ca1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:19:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:19:27Z" name: isvc-primary-147ca1.18c103db16399302 namespace: kserve-ci-e2e-test resourceVersion: "19701" uid: c58f1671-e900-4700-b23e-9a375a9703d3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:19:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-147ca1 namespace: kserve-ci-e2e-test resourceVersion: "19673" uid: b17420f4-e8a9-46e8-8b3a-898ed29bf980 kind: Event lastTimestamp: "2026-07-10T19:19:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-147ca1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:19:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:19:27Z" name: isvc-primary-147ca1.18c103db1639d6b9 namespace: kserve-ci-e2e-test resourceVersion: "19702" uid: 4fe6478f-ed3f-4de6-b2c4-2661476f1fcc reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-147ca1 namespace: kserve-ci-e2e-test resourceVersion: "20317" uid: b17420f4-e8a9-46e8-8b3a-898ed29bf980 kind: Event lastTimestamp: "2026-07-10T19:20:49Z" message: InferenceService [isvc-primary-147ca1] is Ready metadata: creationTimestamp: "2026-07-10T19:20:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:20:49Z" name: isvc-primary-147ca1.18c103ee54812241 namespace: kserve-ci-e2e-test resourceVersion: "20318" uid: 3d13c144-838a-437b-a8f0-41082c9f4925 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:09:30.468828Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8 namespace: kserve-ci-e2e-test resourceVersion: "14564" uid: 79188647-e6db-4318-8c6a-405ae2a43101 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8 to ip-10-0-134-26.ec2.internal metadata: creationTimestamp: "2026-07-10T19:09:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:09:30Z" name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8.18c1035036e7856b namespace: kserve-ci-e2e-test resourceVersion: "14568" uid: d9db01e3-bd6b-46c3-964c-6b9d2e21b95a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c9dd4b6f4-9plfp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:30Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 79188647-e6db-4318-8c6a-405ae2a43101 kind: Event lastTimestamp: "2026-07-10T19:09:30Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-raw-sklearn-batcher-cc766-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:30Z" name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8.18c1035042b8ac41 namespace: kserve-ci-e2e-test resourceVersion: "14579" uid: 71d9ef79-6061-46cb-b970-653958d26d52 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:31Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8 namespace: kserve-ci-e2e-test resourceVersion: "14574" uid: 79188647-e6db-4318-8c6a-405ae2a43101 kind: Event lastTimestamp: "2026-07-10T19:09:31Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:09:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:09:31Z" name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8.18c1035074734251 namespace: kserve-ci-e2e-test resourceVersion: "14586" uid: 9901b8aa-5cf7-4a38-ab6d-20c751822651 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 79188647-e6db-4318-8c6a-405ae2a43101 kind: Event lastTimestamp: "2026-07-10T19:09:31Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7893a590791cc9f33e3a3207dcc1d0843778bbf1aee8b6f22a63e3aa3b91a2e1" already present on machine metadata: creationTimestamp: "2026-07-10T19:09:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:31Z" name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8.18c10350761a7a22 namespace: kserve-ci-e2e-test resourceVersion: "14590" uid: 24b6672e-dee8-467c-8059-e31efb2457af reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 79188647-e6db-4318-8c6a-405ae2a43101 kind: Event lastTimestamp: "2026-07-10T19:09:31Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:09:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:31Z" name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8.18c10350790dd64c namespace: kserve-ci-e2e-test resourceVersion: "14595" uid: 48f8c6d2-2e0e-4639-abda-05357e711fc2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 79188647-e6db-4318-8c6a-405ae2a43101 kind: Event lastTimestamp: "2026-07-10T19:09:31Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:09:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:31Z" name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8.18c103507982e790 namespace: kserve-ci-e2e-test resourceVersion: "14596" uid: 274aba04-ff5c-4f9e-9f73-bd57d59abbeb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 79188647-e6db-4318-8c6a-405ae2a43101 kind: Event lastTimestamp: "2026-07-10T19:09:35Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1722" metadata: creationTimestamp: "2026-07-10T19:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:35Z" name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8.18c103517bf87ad4 namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: 8ef24433-4fb7-495c-9c87-e91906e80292 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 79188647-e6db-4318-8c6a-405ae2a43101 kind: Event lastTimestamp: "2026-07-10T19:09:49Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1722" in 13.476s (13.476s including waiting). Image size: 1555838884 bytes.' metadata: creationTimestamp: "2026-07-10T19:09:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:49Z" name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8.18c103549f421afb namespace: kserve-ci-e2e-test resourceVersion: "14728" uid: 5f48effc-15ec-421f-8559-336cb5512471 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 79188647-e6db-4318-8c6a-405ae2a43101 kind: Event lastTimestamp: "2026-07-10T19:09:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:09:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:49Z" name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8.18c10354a2203232 namespace: kserve-ci-e2e-test resourceVersion: "14729" uid: b2b6f34c-4ecf-4218-8999-795843e3fa30 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 79188647-e6db-4318-8c6a-405ae2a43101 kind: Event lastTimestamp: "2026-07-10T19:09:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:09:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:49Z" name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8.18c10354a28fc50e namespace: kserve-ci-e2e-test resourceVersion: "14730" uid: 595f7a7f-fb90-47f9-82c2-58adcd83090e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 79188647-e6db-4318-8c6a-405ae2a43101 kind: Event lastTimestamp: "2026-07-10T19:09:49Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-10T19:09:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:49Z" name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8.18c10354a29c79ce namespace: kserve-ci-e2e-test resourceVersion: "14731" uid: 939e6111-4cfd-4cac-baeb-c8bc67bf498d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 79188647-e6db-4318-8c6a-405ae2a43101 kind: Event lastTimestamp: "2026-07-10T19:09:51Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.388s (2.388s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-10T19:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:51Z" name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8.18c1035530fed0e9 namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: d0a608b2-0e17-4ca5-9b0c-d95b1309c2ee reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 79188647-e6db-4318-8c6a-405ae2a43101 kind: Event lastTimestamp: "2026-07-10T19:09:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:51Z" name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8.18c1035533f609e7 namespace: kserve-ci-e2e-test resourceVersion: "14755" uid: 21fc5213-0e36-4902-8ff8-101119a7cff3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 79188647-e6db-4318-8c6a-405ae2a43101 kind: Event lastTimestamp: "2026-07-10T19:09:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:51Z" name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8.18c1035534633e28 namespace: kserve-ci-e2e-test resourceVersion: "14756" uid: 79f5afb7-c8d5-4eb1-9a2b-0df59fcb5c80 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 79188647-e6db-4318-8c6a-405ae2a43101 kind: Event lastTimestamp: "2026-07-10T19:09:51Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:4c6a130c213136d803ee987407e3094d900670800a73507fd61c7fd7f0573052" metadata: creationTimestamp: "2026-07-10T19:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:51Z" name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8.18c10355346f2cfb namespace: kserve-ci-e2e-test resourceVersion: "14758" uid: 687cfa28-b21d-4c7c-9058-4a1c3ac72d01 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 79188647-e6db-4318-8c6a-405ae2a43101 kind: Event lastTimestamp: "2026-07-10T19:09:55Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:4c6a130c213136d803ee987407e3094d900670800a73507fd61c7fd7f0573052" in 3.313s (3.313s including waiting). Image size: 239997016 bytes.' metadata: creationTimestamp: "2026-07-10T19:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:55Z" name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8.18c10355f9e9b118 namespace: kserve-ci-e2e-test resourceVersion: "14779" uid: 7153f13a-a2a4-405b-a355-b8ce7cebe3d3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 79188647-e6db-4318-8c6a-405ae2a43101 kind: Event lastTimestamp: "2026-07-10T19:09:55Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-10T19:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:55Z" name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8.18c10355fd8c8c1d namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: 07b3aa5f-2df7-4811-b4f6-1688ad47ebfa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 79188647-e6db-4318-8c6a-405ae2a43101 kind: Event lastTimestamp: "2026-07-10T19:09:55Z" message: Started container agent metadata: creationTimestamp: "2026-07-10T19:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:09:55Z" name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8.18c10355fdfdb851 namespace: kserve-ci-e2e-test resourceVersion: "14781" uid: c5f78954-5f47-47c0-8879-2eee757947db reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-10T19:09:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 79188647-e6db-4318-8c6a-405ae2a43101 kind: Event lastTimestamp: "2026-07-10T19:11:28Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:09:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:28Z" name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8.18c1035629f502de namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: eb2a43ea-14d1-4315-ade6-f830a71f1c91 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-10T19:09:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 79188647-e6db-4318-8c6a-405ae2a43101 kind: Event lastTimestamp: "2026-07-10T19:11:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-10T19:09:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:18Z" name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8.18c1035629ff66f5 namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: c090c333-e14c-4258-8fb4-1c6360c6d62d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 79188647-e6db-4318-8c6a-405ae2a43101 kind: Event lastTimestamp: "2026-07-10T19:11:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:15Z" name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8.18c10368aac297b9 namespace: kserve-ci-e2e-test resourceVersion: "15385" uid: 3a20c35e-6371-4d26-9132-f0e184e4f90c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 79188647-e6db-4318-8c6a-405ae2a43101 kind: Event lastTimestamp: "2026-07-10T19:11:15Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-10T19:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:15Z" name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8.18c10368aac3c37c namespace: kserve-ci-e2e-test resourceVersion: "15389" uid: 034d1d7d-05b9-4649-bba8-29985d0725f0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 79188647-e6db-4318-8c6a-405ae2a43101 kind: Event lastTimestamp: "2026-07-10T19:11:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:15Z" name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8.18c10368aac444fe namespace: kserve-ci-e2e-test resourceVersion: "15393" uid: bf28071b-7443-474b-9e44-59c38e8887fe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:11:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 79188647-e6db-4318-8c6a-405ae2a43101 kind: Event lastTimestamp: "2026-07-10T19:11:26Z" message: 'Readiness probe failed: Get "https://10.133.0.23:8643/healthz": dial tcp 10.133.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:26Z" name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8.18c10368ca45afa2 namespace: kserve-ci-e2e-test resourceVersion: "15569" uid: 50893682-9a6c-412a-92f0-a1848a186524 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8 namespace: kserve-ci-e2e-test resourceVersion: "14553" uid: 74a8b790-51ca-49dd-9f13-e0a0573d709d kind: Event lastTimestamp: "2026-07-10T19:09:30Z" message: 'Created pod: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8-xdjx8' metadata: creationTimestamp: "2026-07-10T19:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:09:30Z" name: isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8.18c103503684b95d namespace: kserve-ci-e2e-test resourceVersion: "14565" uid: fe2514c2-6993-41ec-b1bd-8d5930161dd0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-cc766-predictor namespace: kserve-ci-e2e-test resourceVersion: "14552" uid: f8ab0ee7-02eb-4e36-92b9-7169b78f7f85 kind: Event lastTimestamp: "2026-07-10T19:09:30Z" message: Scaled up replica set isvc-raw-sklearn-batcher-cc766-predictor-6fcdcdf9d8 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:09:30Z" name: isvc-raw-sklearn-batcher-cc766-predictor.18c103502e192b0c namespace: kserve-ci-e2e-test resourceVersion: "14554" uid: f1bba0f0-9490-48fb-9fa3-57f30e724db1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-10T19:09:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-cc766-predictor namespace: kserve-ci-e2e-test resourceVersion: "14559" uid: cf699e92-baa1-4f08-853d-567c5f8b9ef0 kind: Event lastTimestamp: "2026-07-10T19:10:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:09:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:10:45Z" name: isvc-raw-sklearn-batcher-cc766-predictor.18c10353b3811375 namespace: kserve-ci-e2e-test resourceVersion: "15146" uid: 56c063a3-44ef-42ab-ac40-9ebb0fb14456 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-10T19:09:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-cc766-predictor namespace: kserve-ci-e2e-test resourceVersion: "14559" uid: cf699e92-baa1-4f08-853d-567c5f8b9ef0 kind: Event lastTimestamp: "2026-07-10T19:10:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:09:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:10:45Z" name: isvc-raw-sklearn-batcher-cc766-predictor.18c10353b3e04e71 namespace: kserve-ci-e2e-test resourceVersion: "15147" uid: 3bf8724a-a7e7-430e-ac6b-cb68f09b1536 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-cc766-predictor namespace: kserve-ci-e2e-test resourceVersion: "14700" uid: cf699e92-baa1-4f08-853d-567c5f8b9ef0 kind: Event lastTimestamp: "2026-07-10T19:11:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:00Z" name: isvc-raw-sklearn-batcher-cc766-predictor.18c103652f28f5c8 namespace: kserve-ci-e2e-test resourceVersion: "15249" uid: 364530c6-7db7-4496-894a-e02346f553d0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-cc766-predictor namespace: kserve-ci-e2e-test resourceVersion: "14700" uid: cf699e92-baa1-4f08-853d-567c5f8b9ef0 kind: Event lastTimestamp: "2026-07-10T19:11:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:00Z" name: isvc-raw-sklearn-batcher-cc766-predictor.18c103652f8cc904 namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: 99392eb2-8e2b-4438-ab3b-ec8d71aca4b2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-10T19:09:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-cc766 namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: 72ab7d06-cf4c-487e-9b2a-56e9d51a0f6e kind: Event lastTimestamp: "2026-07-10T19:11:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:11:08Z" name: isvc-raw-sklearn-batcher-cc766.18c10350344bbd14 namespace: kserve-ci-e2e-test resourceVersion: "15311" uid: da4c386e-155d-4043-8e7c-fb8b2d318653 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-cc766 namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: 72ab7d06-cf4c-487e-9b2a-56e9d51a0f6e kind: Event lastTimestamp: "2026-07-10T19:09:30Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-cc766": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-cc766": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:09:30Z" name: isvc-raw-sklearn-batcher-cc766.18c103503497abd1 namespace: kserve-ci-e2e-test resourceVersion: "14561" uid: 1a7c02ed-886a-435b-b3c8-25ae8de78071 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:09:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-cc766 namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: 72ab7d06-cf4c-487e-9b2a-56e9d51a0f6e kind: Event lastTimestamp: "2026-07-10T19:09:30Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-cc766": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:09:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:09:30Z" name: isvc-raw-sklearn-batcher-cc766.18c103503497f219 namespace: kserve-ci-e2e-test resourceVersion: "14562" uid: e12c0bc1-31f4-4df2-bce2-6afc06b4129a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-cc766 namespace: kserve-ci-e2e-test resourceVersion: "15309" uid: 72ab7d06-cf4c-487e-9b2a-56e9d51a0f6e kind: Event lastTimestamp: "2026-07-10T19:11:08Z" message: InferenceService [isvc-raw-sklearn-batcher-cc766] is Ready metadata: creationTimestamp: "2026-07-10T19:11:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:11:08Z" name: isvc-raw-sklearn-batcher-cc766.18c10366f4cff98b namespace: kserve-ci-e2e-test resourceVersion: "15310" uid: 6bd6a8aa-bcab-4c55-9af4-223c480fc8dd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:20:57.178092Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-147ca1-predictor-5d9994b64b-28qqq namespace: kserve-ci-e2e-test resourceVersion: "20401" uid: ea4ff367-de06-45b5-88e0-3ae4b0ef27e8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-147ca1-predictor-5d9994b64b-28qqq to ip-10-0-134-26.ec2.internal metadata: creationTimestamp: "2026-07-10T19:20:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:20:57Z" name: isvc-secondary-147ca1-predictor-5d9994b64b-28qqq.18c103f019f75412 namespace: kserve-ci-e2e-test resourceVersion: "20405" uid: 30c5f208-f1f4-42b4-92cd-90b4f2930a44 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c9dd4b6f4-9plfp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:57Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-147ca1-predictor-5d9994b64b-28qqq namespace: kserve-ci-e2e-test resourceVersion: "20411" uid: ea4ff367-de06-45b5-88e0-3ae4b0ef27e8 kind: Event lastTimestamp: "2026-07-10T19:20:57Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:20:57Z" name: isvc-secondary-147ca1-predictor-5d9994b64b-28qqq.18c103f0335a5efb namespace: kserve-ci-e2e-test resourceVersion: "20420" uid: 57b7244a-e60c-4f3b-a63e-245ab990bd4d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-147ca1-predictor-5d9994b64b-28qqq namespace: kserve-ci-e2e-test resourceVersion: "20403" uid: ea4ff367-de06-45b5-88e0-3ae4b0ef27e8 kind: Event lastTimestamp: "2026-07-10T19:21:03Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7893a590791cc9f33e3a3207dcc1d0843778bbf1aee8b6f22a63e3aa3b91a2e1" already present on machine metadata: creationTimestamp: "2026-07-10T19:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:03Z" name: isvc-secondary-147ca1-predictor-5d9994b64b-28qqq.18c103f034bf01f8 namespace: kserve-ci-e2e-test resourceVersion: "20470" uid: 5e7c45c2-b74b-40f6-87bb-a7bb67fe6019 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-147ca1-predictor-5d9994b64b-28qqq namespace: kserve-ci-e2e-test resourceVersion: "20403" uid: ea4ff367-de06-45b5-88e0-3ae4b0ef27e8 kind: Event lastTimestamp: "2026-07-10T19:21:03Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:03Z" name: isvc-secondary-147ca1-predictor-5d9994b64b-28qqq.18c103f0380c9ed5 namespace: kserve-ci-e2e-test resourceVersion: "20472" uid: f8949a1d-ee0e-4694-bbcf-359d44cdf18b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-147ca1-predictor-5d9994b64b-28qqq namespace: kserve-ci-e2e-test resourceVersion: "20403" uid: ea4ff367-de06-45b5-88e0-3ae4b0ef27e8 kind: Event lastTimestamp: "2026-07-10T19:21:03Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:03Z" name: isvc-secondary-147ca1-predictor-5d9994b64b-28qqq.18c103f03896887b namespace: kserve-ci-e2e-test resourceVersion: "20473" uid: 6e2aed03-df84-4a43-a41a-0122b10f5c83 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-147ca1-predictor-5d9994b64b-28qqq namespace: kserve-ci-e2e-test resourceVersion: "20403" uid: ea4ff367-de06-45b5-88e0-3ae4b0ef27e8 kind: Event lastTimestamp: "2026-07-10T19:21:09Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-147ca1-predictor-5d9994b64b-28qqq_kserve-ci-e2e-test(ea4ff367-de06-45b5-88e0-3ae4b0ef27e8) metadata: creationTimestamp: "2026-07-10T19:21:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:09Z" name: isvc-secondary-147ca1-predictor-5d9994b64b-28qqq.18c103f30d573330 namespace: kserve-ci-e2e-test resourceVersion: "20523" uid: 621a56fb-1369-4c24-b24c-26bf54a48b54 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-147ca1-predictor-5d9994b64b namespace: kserve-ci-e2e-test resourceVersion: "20391" uid: f354e9a4-f319-4aef-8612-a4ee33498123 kind: Event lastTimestamp: "2026-07-10T19:20:57Z" message: 'Created pod: isvc-secondary-147ca1-predictor-5d9994b64b-28qqq' metadata: creationTimestamp: "2026-07-10T19:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:57Z" name: isvc-secondary-147ca1-predictor-5d9994b64b.18c103f01995ae79 namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 51a1a60c-9524-491a-84fa-530ffba930fd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-147ca1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20389" uid: a46cb954-a8f3-4097-8701-ae06a696c54b kind: Event lastTimestamp: "2026-07-10T19:20:57Z" message: Scaled up replica set isvc-secondary-147ca1-predictor-5d9994b64b from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:57Z" name: isvc-secondary-147ca1-predictor.18c103f0153d503e namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 8bd6a461-7976-4714-a25c-d273e7985f40 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-147ca1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20390" uid: 49526e60-c9aa-4b84-b40b-ebe0d68a207c kind: Event lastTimestamp: "2026-07-10T19:21:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:21:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:12Z" name: isvc-secondary-147ca1-predictor.18c103f393a0b596 namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: 167d3019-ae33-4114-abdf-840d15b44a01 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-147ca1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20390" uid: 49526e60-c9aa-4b84-b40b-ebe0d68a207c kind: Event lastTimestamp: "2026-07-10T19:21:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:21:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:12Z" name: isvc-secondary-147ca1-predictor.18c103f39409affd namespace: kserve-ci-e2e-test resourceVersion: "20548" uid: 680ec4e3-aa95-46ac-9221-a5778e519bc3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-07-10T19:20:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-147ca1 namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: 08024825-50ef-4199-84e1-fffbc4db1e77 kind: Event lastTimestamp: "2026-07-10T19:21:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:21:10Z" name: isvc-secondary-147ca1.18c103f01540de06 namespace: kserve-ci-e2e-test resourceVersion: "20530" uid: e223d3a0-1c51-4c4e-b933-f06b2b10d616 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:20:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-147ca1 namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: 08024825-50ef-4199-84e1-fffbc4db1e77 kind: Event lastTimestamp: "2026-07-10T19:20:57Z" message: 'Failed to update status for InferenceService "isvc-secondary-147ca1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-147ca1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:20:57Z" name: isvc-secondary-147ca1.18c103f01593227e namespace: kserve-ci-e2e-test resourceVersion: "20410" uid: aafe1c26-5bff-4219-94b6-e511c5773cfe reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:20:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-147ca1 namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: 08024825-50ef-4199-84e1-fffbc4db1e77 kind: Event lastTimestamp: "2026-07-10T19:20:57Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-147ca1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:20:57Z" name: isvc-secondary-147ca1.18c103f015937551 namespace: kserve-ci-e2e-test resourceVersion: "20412" uid: 4b277794-114e-4dc8-a43f-98812c013593 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:11:15.715278Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 168d15a7-e849-4968-bb60-e19259c523a3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf to ip-10-0-134-26.ec2.internal metadata: creationTimestamp: "2026-07-10T19:11:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:11:15Z" name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf.18c10368b814e4f3 namespace: kserve-ci-e2e-test resourceVersion: "15419" uid: 8f33ddde-a33b-4254-ba52-fe8b918d4994 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c9dd4b6f4-9plfp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:16Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf namespace: kserve-ci-e2e-test resourceVersion: "15430" uid: 168d15a7-e849-4968-bb60-e19259c523a3 kind: Event lastTimestamp: "2026-07-10T19:11:16Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:11:16Z" name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf.18c10368d15f7fb7 namespace: kserve-ci-e2e-test resourceVersion: "15456" uid: d4e12db3-a46b-4253-b77a-df610f2e2832 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: 168d15a7-e849-4968-bb60-e19259c523a3 kind: Event lastTimestamp: "2026-07-10T19:11:16Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7893a590791cc9f33e3a3207dcc1d0843778bbf1aee8b6f22a63e3aa3b91a2e1" already present on machine metadata: creationTimestamp: "2026-07-10T19:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:16Z" name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf.18c10368d2b72031 namespace: kserve-ci-e2e-test resourceVersion: "15458" uid: f3641675-4c5b-4f8d-a2f9-daa0806e13ae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: 168d15a7-e849-4968-bb60-e19259c523a3 kind: Event lastTimestamp: "2026-07-10T19:11:16Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:16Z" name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf.18c10368d545e3aa namespace: kserve-ci-e2e-test resourceVersion: "15459" uid: b84fd90d-dd5d-49de-948e-d03511b40c4b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: 168d15a7-e849-4968-bb60-e19259c523a3 kind: Event lastTimestamp: "2026-07-10T19:11:16Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:16Z" name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf.18c10368d5a70aac namespace: kserve-ci-e2e-test resourceVersion: "15460" uid: 98d9dbf9-aa77-4244-be5b-36c31c9ddb48 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: 168d15a7-e849-4968-bb60-e19259c523a3 kind: Event lastTimestamp: "2026-07-10T19:11:20Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:11:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:20Z" name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf.18c10369cd1cae82 namespace: kserve-ci-e2e-test resourceVersion: "15512" uid: 095aa6ca-6dd4-45dd-b5c8-c46f90e7b42e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: 168d15a7-e849-4968-bb60-e19259c523a3 kind: Event lastTimestamp: "2026-07-10T19:11:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:11:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:20Z" name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf.18c10369cfeb42fd namespace: kserve-ci-e2e-test resourceVersion: "15515" uid: fe2421de-6769-4cb3-bfef-15bd7b6f2ebe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: 168d15a7-e849-4968-bb60-e19259c523a3 kind: Event lastTimestamp: "2026-07-10T19:11:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:11:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:20Z" name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf.18c10369d0572da8 namespace: kserve-ci-e2e-test resourceVersion: "15516" uid: 85498a5e-fa9f-4cdc-8f31-fdd1e8f4e797 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: 168d15a7-e849-4968-bb60-e19259c523a3 kind: Event lastTimestamp: "2026-07-10T19:11:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:11:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:20Z" name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf.18c10369d0621091 namespace: kserve-ci-e2e-test resourceVersion: "15517" uid: fb45a608-b052-4b9d-ba56-e2a948a1c2db reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: 168d15a7-e849-4968-bb60-e19259c523a3 kind: Event lastTimestamp: "2026-07-10T19:11:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:11:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:20Z" name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf.18c10369d344a500 namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: 184e56b2-4b8f-457a-a3b2-b787099c7475 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: 168d15a7-e849-4968-bb60-e19259c523a3 kind: Event lastTimestamp: "2026-07-10T19:11:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:11:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:20Z" name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf.18c10369d3af2fff namespace: kserve-ci-e2e-test resourceVersion: "15522" uid: 2425ea70-3165-4297-97e8-a61dbd0a3663 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-10T19:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: 168d15a7-e849-4968-bb60-e19259c523a3 kind: Event lastTimestamp: "2026-07-10T19:13:08Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:08Z" name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf.18c1036a09282126 namespace: kserve-ci-e2e-test resourceVersion: "16535" uid: f9bd2e79-9f2b-4f17-b706-8c7500d02c55 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: 168d15a7-e849-4968-bb60-e19259c523a3 kind: Event lastTimestamp: "2026-07-10T19:13:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:13:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:05Z" name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf.18c1038261b0bd7b namespace: kserve-ci-e2e-test resourceVersion: "16422" uid: 654b8036-5a78-4c13-acf7-0d0442319927 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: 168d15a7-e849-4968-bb60-e19259c523a3 kind: Event lastTimestamp: "2026-07-10T19:13:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:13:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:05Z" name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf.18c1038261b16477 namespace: kserve-ci-e2e-test resourceVersion: "16424" uid: b99ee8c0-e2ac-4a2b-89f3-63911e671d37 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: 168d15a7-e849-4968-bb60-e19259c523a3 kind: Event lastTimestamp: "2026-07-10T19:13:07Z" message: 'Readiness probe failed: Get "https://10.133.0.24:8643/healthz": dial tcp 10.133.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:13:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:07Z" name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf.18c10382b7963e5e namespace: kserve-ci-e2e-test resourceVersion: "16527" uid: c3d6a46d-6b48-40b6-a1d2-4de921ab9367 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4 namespace: kserve-ci-e2e-test resourceVersion: "15395" uid: 4ba29090-d650-4833-b8de-d5a48814f008 kind: Event lastTimestamp: "2026-07-10T19:11:15Z" message: 'Created pod: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4-drbvf' metadata: creationTimestamp: "2026-07-10T19:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:15Z" name: isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4.18c10368b796c2eb namespace: kserve-ci-e2e-test resourceVersion: "15413" uid: c2a6b03c-b8e6-4866-b1f6-acb78f950a62 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-e3394-predictor namespace: kserve-ci-e2e-test resourceVersion: "15391" uid: 91635917-89a0-4ed3-98b9-b7dd771f5f07 kind: Event lastTimestamp: "2026-07-10T19:11:15Z" message: Scaled up replica set isvc-sklearn-graph-raw-e3394-predictor-6db968d9b4 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:15Z" name: isvc-sklearn-graph-raw-e3394-predictor.18c10368adbcd132 namespace: kserve-ci-e2e-test resourceVersion: "15398" uid: 3118669d-5f85-49ec-b756-c8da3f2bfc16 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-e3394-predictor namespace: kserve-ci-e2e-test resourceVersion: "15394" uid: 5c28e5ff-c9d6-4832-a329-6598795b0ef8 kind: Event lastTimestamp: "2026-07-10T19:11:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:11:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:30Z" name: isvc-sklearn-graph-raw-e3394-predictor.18c1036c2c626b46 namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 3f55ef9c-88c7-4832-a366-f96d8e47631d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-e3394-predictor namespace: kserve-ci-e2e-test resourceVersion: "15394" uid: 5c28e5ff-c9d6-4832-a329-6598795b0ef8 kind: Event lastTimestamp: "2026-07-10T19:11:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:11:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:30Z" name: isvc-sklearn-graph-raw-e3394-predictor.18c1036c2cc770ce namespace: kserve-ci-e2e-test resourceVersion: "15602" uid: 0aca1255-058f-4b42-97f6-f048742c4fb5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:11:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-e3394-predictor namespace: kserve-ci-e2e-test resourceVersion: "15600" uid: 5c28e5ff-c9d6-4832-a329-6598795b0ef8 kind: Event lastTimestamp: "2026-07-10T19:13:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:13:00Z" name: isvc-sklearn-graph-raw-e3394-predictor.18c1036fabe3e5cd namespace: kserve-ci-e2e-test resourceVersion: "16323" uid: aefc0b68-d792-431b-8887-1611295b92c0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:11:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-e3394-predictor namespace: kserve-ci-e2e-test resourceVersion: "15600" uid: 5c28e5ff-c9d6-4832-a329-6598795b0ef8 kind: Event lastTimestamp: "2026-07-10T19:13:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:13:00Z" name: isvc-sklearn-graph-raw-e3394-predictor.18c1036fac48aa69 namespace: kserve-ci-e2e-test resourceVersion: "16324" uid: 03d27d76-163b-447f-bb88-635248f080a6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-10T19:11:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-e3394 namespace: kserve-ci-e2e-test resourceVersion: "15365" uid: bdf2cc8a-357d-47f9-8118-c5f3a1a0dadf kind: Event lastTimestamp: "2026-07-10T19:12:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:12:38Z" name: isvc-sklearn-graph-raw-e3394.18c10368ade13067 namespace: kserve-ci-e2e-test resourceVersion: "16123" uid: 285fb1aa-c230-44ca-8611-b373185207d4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-e3394 namespace: kserve-ci-e2e-test resourceVersion: "15365" uid: bdf2cc8a-357d-47f9-8118-c5f3a1a0dadf kind: Event lastTimestamp: "2026-07-10T19:11:15Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-e3394": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-e3394": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:11:15Z" name: isvc-sklearn-graph-raw-e3394.18c10368aecf0fec namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: 0b32f5ca-8cee-4c2e-9b55-530572ebf4b3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:12:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-e3394 namespace: kserve-ci-e2e-test resourceVersion: "16121" uid: bdf2cc8a-357d-47f9-8118-c5f3a1a0dadf kind: Event lastTimestamp: "2026-07-10T19:12:38Z" message: InferenceService [isvc-sklearn-graph-raw-e3394] is Ready metadata: creationTimestamp: "2026-07-10T19:12:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:12:38Z" name: isvc-sklearn-graph-raw-e3394.18c1037bfe6152fc namespace: kserve-ci-e2e-test resourceVersion: "16122" uid: 1634157b-97fc-4572-9d52-4075c50f79ba reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:13:06.041423Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz namespace: kserve-ci-e2e-test resourceVersion: "16431" uid: e7201858-6980-453d-8745-c8ff65e9a191 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz to ip-10-0-134-26.ec2.internal metadata: creationTimestamp: "2026-07-10T19:13:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:13:06Z" name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz.18c103826808442e namespace: kserve-ci-e2e-test resourceVersion: "16439" uid: 13d3c309-46d8-4561-8f69-3ac51293ea24 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c9dd4b6f4-9plfp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:06Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz namespace: kserve-ci-e2e-test resourceVersion: "16451" uid: e7201858-6980-453d-8745-c8ff65e9a191 kind: Event lastTimestamp: "2026-07-10T19:13:06Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:13:06Z" name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz.18c1038282040c1f namespace: kserve-ci-e2e-test resourceVersion: "16499" uid: 9b0f542e-5f4f-4dd2-940d-e512d5328706 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz namespace: kserve-ci-e2e-test resourceVersion: "16436" uid: e7201858-6980-453d-8745-c8ff65e9a191 kind: Event lastTimestamp: "2026-07-10T19:13:06Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7893a590791cc9f33e3a3207dcc1d0843778bbf1aee8b6f22a63e3aa3b91a2e1" already present on machine metadata: creationTimestamp: "2026-07-10T19:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:06Z" name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz.18c103828377db2f namespace: kserve-ci-e2e-test resourceVersion: "16502" uid: 4a5727d4-be58-4bb7-9d1a-008f17e9880d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz namespace: kserve-ci-e2e-test resourceVersion: "16436" uid: e7201858-6980-453d-8745-c8ff65e9a191 kind: Event lastTimestamp: "2026-07-10T19:13:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:06Z" name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz.18c10382865ba253 namespace: kserve-ci-e2e-test resourceVersion: "16503" uid: 2e3f02af-bc7e-4a0f-b679-7018e094c95b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz namespace: kserve-ci-e2e-test resourceVersion: "16436" uid: e7201858-6980-453d-8745-c8ff65e9a191 kind: Event lastTimestamp: "2026-07-10T19:13:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:06Z" name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz.18c10382872ba56f namespace: kserve-ci-e2e-test resourceVersion: "16504" uid: 0df4239d-0408-4e04-a94b-db1411f4a648 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz namespace: kserve-ci-e2e-test resourceVersion: "16436" uid: e7201858-6980-453d-8745-c8ff65e9a191 kind: Event lastTimestamp: "2026-07-10T19:13:10Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:10Z" name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz.18c103838758ff88 namespace: kserve-ci-e2e-test resourceVersion: "16555" uid: 5e83fe1e-acf4-43f4-ac1a-89a35048d181 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz namespace: kserve-ci-e2e-test resourceVersion: "16436" uid: e7201858-6980-453d-8745-c8ff65e9a191 kind: Event lastTimestamp: "2026-07-10T19:13:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:10Z" name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz.18c103838f273830 namespace: kserve-ci-e2e-test resourceVersion: "16565" uid: 1492df3a-9831-4619-af9f-a2cedf80fb6c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz namespace: kserve-ci-e2e-test resourceVersion: "16436" uid: e7201858-6980-453d-8745-c8ff65e9a191 kind: Event lastTimestamp: "2026-07-10T19:13:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:11Z" name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz.18c103838fa5a663 namespace: kserve-ci-e2e-test resourceVersion: "16568" uid: 1992b85c-938b-46a2-9569-4ef6381bddc2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz namespace: kserve-ci-e2e-test resourceVersion: "16436" uid: e7201858-6980-453d-8745-c8ff65e9a191 kind: Event lastTimestamp: "2026-07-10T19:13:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:11Z" name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz.18c103838fb20114 namespace: kserve-ci-e2e-test resourceVersion: "16569" uid: 0587cdab-d07d-439a-83c4-858108eac8da reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz namespace: kserve-ci-e2e-test resourceVersion: "16436" uid: e7201858-6980-453d-8745-c8ff65e9a191 kind: Event lastTimestamp: "2026-07-10T19:13:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:11Z" name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz.18c10383938cf596 namespace: kserve-ci-e2e-test resourceVersion: "16575" uid: fbe5c12b-4807-4c45-abf5-4c2ade752ec2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz namespace: kserve-ci-e2e-test resourceVersion: "16436" uid: e7201858-6980-453d-8745-c8ff65e9a191 kind: Event lastTimestamp: "2026-07-10T19:13:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:11Z" name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz.18c10383942dc606 namespace: kserve-ci-e2e-test resourceVersion: "16578" uid: 78169c35-a4ec-4536-b10a-fa1563a78537 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-10T19:13:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz namespace: kserve-ci-e2e-test resourceVersion: "16436" uid: e7201858-6980-453d-8745-c8ff65e9a191 kind: Event lastTimestamp: "2026-07-10T19:14:58Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:13:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:14:58Z" name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz.18c10383ffb2cc27 namespace: kserve-ci-e2e-test resourceVersion: "17533" uid: 1a5f77c3-1309-40c9-b015-e4e04033d831 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz namespace: kserve-ci-e2e-test resourceVersion: "16436" uid: e7201858-6980-453d-8745-c8ff65e9a191 kind: Event lastTimestamp: "2026-07-10T19:14:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:14:56Z" name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz.18c1039c112cabe8 namespace: kserve-ci-e2e-test resourceVersion: "17457" uid: 08b00689-d19f-4df9-bb26-faa0707fd7b0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz namespace: kserve-ci-e2e-test resourceVersion: "16436" uid: e7201858-6980-453d-8745-c8ff65e9a191 kind: Event lastTimestamp: "2026-07-10T19:14:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:14:56Z" name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz.18c1039c112d1dfd namespace: kserve-ci-e2e-test resourceVersion: "17458" uid: 4d712f8f-e169-41c5-9814-8c96d066f21f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz namespace: kserve-ci-e2e-test resourceVersion: "16436" uid: e7201858-6980-453d-8745-c8ff65e9a191 kind: Event lastTimestamp: "2026-07-10T19:14:58Z" message: 'Readiness probe failed: Get "https://10.133.0.27:8643/healthz": dial tcp 10.133.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:14:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:14:58Z" name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz.18c1039cadfb8884 namespace: kserve-ci-e2e-test resourceVersion: "17530" uid: 4edc53d6-e7b3-4c3b-bcbc-a507fb98ea18 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84 namespace: kserve-ci-e2e-test resourceVersion: "16411" uid: 30423436-6188-4f5d-a2b2-e2fd8778df86 kind: Event lastTimestamp: "2026-07-10T19:13:06Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84-rcdqz' metadata: creationTimestamp: "2026-07-10T19:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:13:06Z" name: isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84.18c10382677b8197 namespace: kserve-ci-e2e-test resourceVersion: "16434" uid: 584e0d2f-adaa-42a8-9967-b142797cec9d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-fed9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16406" uid: a2312f6e-6974-4d22-97da-d8d247ba4f26 kind: Event lastTimestamp: "2026-07-10T19:13:05Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-fed9d-predictor-67ccfccd84 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:13:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:13:05Z" name: isvc-sklearn-graph-raw-hpa-fed9d-predictor.18c103825f5f9634 namespace: kserve-ci-e2e-test resourceVersion: "16416" uid: 2070860e-2439-4fd3-89bc-f7a94237dfde reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:13:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-fed9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16410" uid: ee46d958-f17d-4011-801a-155d90b611cb kind: Event lastTimestamp: "2026-07-10T19:13:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:13:50Z" name: isvc-sklearn-graph-raw-hpa-fed9d-predictor.18c10385dd5b62c2 namespace: kserve-ci-e2e-test resourceVersion: "16883" uid: 6d70fdd9-46d1-4381-98da-360d31003520 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:13:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-fed9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16410" uid: ee46d958-f17d-4011-801a-155d90b611cb kind: Event lastTimestamp: "2026-07-10T19:13:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:13:50Z" name: isvc-sklearn-graph-raw-hpa-fed9d-predictor.18c10385ddea5746 namespace: kserve-ci-e2e-test resourceVersion: "16884" uid: 09710b90-4c96-458b-95d7-6e75f2ff07f1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:14:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-fed9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16666" uid: ee46d958-f17d-4011-801a-155d90b611cb kind: Event lastTimestamp: "2026-07-10T19:14:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:14:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:14:50Z" name: isvc-sklearn-graph-raw-hpa-fed9d-predictor.18c103905a8180a4 namespace: kserve-ci-e2e-test resourceVersion: "17366" uid: 15e9507c-f728-44e9-bc19-ce349ea3e5e2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:14:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-fed9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16666" uid: ee46d958-f17d-4011-801a-155d90b611cb kind: Event lastTimestamp: "2026-07-10T19:14:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:14:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:14:51Z" name: isvc-sklearn-graph-raw-hpa-fed9d-predictor.18c103905aeaa78a namespace: kserve-ci-e2e-test resourceVersion: "17367" uid: 008733a4-f55d-4e49-8032-3781729721ce reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-10T19:13:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-fed9d namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: 4ebd107a-3052-4a80-97b5-2e45d891dec2 kind: Event lastTimestamp: "2026-07-10T19:14:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:13:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:14:29Z" name: isvc-sklearn-graph-raw-hpa-fed9d.18c103825eb9a9d8 namespace: kserve-ci-e2e-test resourceVersion: "17177" uid: 3aa33a73-b02a-42be-868e-54bb3c935632 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-fed9d namespace: kserve-ci-e2e-test resourceVersion: "17173" uid: 4ebd107a-3052-4a80-97b5-2e45d891dec2 kind: Event lastTimestamp: "2026-07-10T19:14:28Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-fed9d] is Ready metadata: creationTimestamp: "2026-07-10T19:14:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:14:28Z" name: isvc-sklearn-graph-raw-hpa-fed9d.18c10395b785ebc6 namespace: kserve-ci-e2e-test resourceVersion: "17174" uid: 617c41ec-6dc8-4cb9-9409-ef004089bcae reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:16:41.655844Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm namespace: kserve-ci-e2e-test resourceVersion: "18414" uid: dd52b2b9-efec-49df-8452-b38247a60cc2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm to ip-10-0-134-26.ec2.internal metadata: creationTimestamp: "2026-07-10T19:16:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:16:41Z" name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm.18c103b49ba78c26 namespace: kserve-ci-e2e-test resourceVersion: "18419" uid: d6a89080-ccef-4ceb-bfd3-f43e0410cd87 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c9dd4b6f4-9plfp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:16:42Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm namespace: kserve-ci-e2e-test resourceVersion: "18435" uid: dd52b2b9-efec-49df-8452-b38247a60cc2 kind: Event lastTimestamp: "2026-07-10T19:16:42Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:16:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:16:42Z" name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm.18c103b4b5a9653e namespace: kserve-ci-e2e-test resourceVersion: "18450" uid: cec36550-4f98-41af-ad24-90456a9cde45 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:16:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm namespace: kserve-ci-e2e-test resourceVersion: "18415" uid: dd52b2b9-efec-49df-8452-b38247a60cc2 kind: Event lastTimestamp: "2026-07-10T19:16:42Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7893a590791cc9f33e3a3207dcc1d0843778bbf1aee8b6f22a63e3aa3b91a2e1" already present on machine metadata: creationTimestamp: "2026-07-10T19:16:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:16:42Z" name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm.18c103b4b719084e namespace: kserve-ci-e2e-test resourceVersion: "18452" uid: d2ebdbe5-5f36-42a2-b698-ce8c47a8191d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:16:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm namespace: kserve-ci-e2e-test resourceVersion: "18415" uid: dd52b2b9-efec-49df-8452-b38247a60cc2 kind: Event lastTimestamp: "2026-07-10T19:16:42Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:16:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:16:42Z" name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm.18c103b4b99cbfcb namespace: kserve-ci-e2e-test resourceVersion: "18454" uid: cb64c7b9-1744-4ae0-8bec-773430db1498 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:16:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm namespace: kserve-ci-e2e-test resourceVersion: "18415" uid: dd52b2b9-efec-49df-8452-b38247a60cc2 kind: Event lastTimestamp: "2026-07-10T19:16:42Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:16:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:16:42Z" name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm.18c103b4ba0fa0b0 namespace: kserve-ci-e2e-test resourceVersion: "18455" uid: ec3f7244-9e1e-42ec-820c-ffc4938e5dd6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:16:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm namespace: kserve-ci-e2e-test resourceVersion: "18415" uid: dd52b2b9-efec-49df-8452-b38247a60cc2 kind: Event lastTimestamp: "2026-07-10T19:16:46Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:16:46Z" name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm.18c103b5cea2479c namespace: kserve-ci-e2e-test resourceVersion: "18497" uid: aae6e81f-3511-4772-9d98-0fb81a545dd2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:16:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm namespace: kserve-ci-e2e-test resourceVersion: "18415" uid: dd52b2b9-efec-49df-8452-b38247a60cc2 kind: Event lastTimestamp: "2026-07-10T19:16:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:16:46Z" name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm.18c103b5d188d994 namespace: kserve-ci-e2e-test resourceVersion: "18499" uid: 640e2476-c671-4329-b932-b8e523ac011b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:16:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm namespace: kserve-ci-e2e-test resourceVersion: "18415" uid: dd52b2b9-efec-49df-8452-b38247a60cc2 kind: Event lastTimestamp: "2026-07-10T19:16:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:16:46Z" name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm.18c103b5d206519f namespace: kserve-ci-e2e-test resourceVersion: "18502" uid: ac1ad0d6-a738-4a09-969a-c226ba252b77 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:16:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm namespace: kserve-ci-e2e-test resourceVersion: "18415" uid: dd52b2b9-efec-49df-8452-b38247a60cc2 kind: Event lastTimestamp: "2026-07-10T19:16:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:16:46Z" name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm.18c103b5d2120c1d namespace: kserve-ci-e2e-test resourceVersion: "18503" uid: 55c19b25-0d2a-4b49-a17a-511c0facbcfd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:16:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm namespace: kserve-ci-e2e-test resourceVersion: "18415" uid: dd52b2b9-efec-49df-8452-b38247a60cc2 kind: Event lastTimestamp: "2026-07-10T19:16:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:16:46Z" name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm.18c103b5d56f9a7e namespace: kserve-ci-e2e-test resourceVersion: "18505" uid: 8b509efa-3fe9-4aab-8aa4-5c179641898a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:16:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm namespace: kserve-ci-e2e-test resourceVersion: "18415" uid: dd52b2b9-efec-49df-8452-b38247a60cc2 kind: Event lastTimestamp: "2026-07-10T19:16:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:16:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:16:46Z" name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm.18c103b5d5e457c3 namespace: kserve-ci-e2e-test resourceVersion: "18506" uid: 0c9f024d-7027-4dad-9fa6-60f5cb8af9e1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 19 eventTime: null firstTimestamp: "2026-07-10T19:16:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm namespace: kserve-ci-e2e-test resourceVersion: "18415" uid: dd52b2b9-efec-49df-8452-b38247a60cc2 kind: Event lastTimestamp: "2026-07-10T19:19:29Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:16:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:29Z" name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm.18c103b60ac5fc4f namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: d4335ddd-3e08-4964-b28b-9fc970bff519 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm namespace: kserve-ci-e2e-test resourceVersion: "18415" uid: dd52b2b9-efec-49df-8452-b38247a60cc2 kind: Event lastTimestamp: "2026-07-10T19:19:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:19:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:21Z" name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm.18c103d9e719f8b2 namespace: kserve-ci-e2e-test resourceVersion: "19627" uid: 2be1ce76-7720-4410-b109-c3e447d04c37 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm namespace: kserve-ci-e2e-test resourceVersion: "18415" uid: dd52b2b9-efec-49df-8452-b38247a60cc2 kind: Event lastTimestamp: "2026-07-10T19:19:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:19:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:21Z" name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm.18c103d9e720b266 namespace: kserve-ci-e2e-test resourceVersion: "19628" uid: 1fa21e1d-f596-40ae-b4d6-5858f4b0ce82 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:19:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm namespace: kserve-ci-e2e-test resourceVersion: "18415" uid: dd52b2b9-efec-49df-8452-b38247a60cc2 kind: Event lastTimestamp: "2026-07-10T19:19:28Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:19:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:28Z" name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm.18c103da5d42e4dc namespace: kserve-ci-e2e-test resourceVersion: "19730" uid: d0564a25-cbc0-40e8-98de-5418ebfb7489 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:16:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b namespace: kserve-ci-e2e-test resourceVersion: "18395" uid: 96e1dbe6-e591-47bf-82ba-9aeb79ef8ee8 kind: Event lastTimestamp: "2026-07-10T19:16:41Z" message: 'Created pod: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b-8r5bm' metadata: creationTimestamp: "2026-07-10T19:16:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:16:41Z" name: isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b.18c103b49b3cba52 namespace: kserve-ci-e2e-test resourceVersion: "18416" uid: 9ca9a897-2988-4afb-9da9-cb45fe6f669b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:16:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-dc419-predictor namespace: kserve-ci-e2e-test resourceVersion: "18387" uid: e55e6c1e-2612-4f70-9d1f-da9971d812c0 kind: Event lastTimestamp: "2026-07-10T19:16:41Z" message: Scaled up replica set isvc-sklearn-scale-raw-dc419-predictor-7ddf54994b from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:16:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:16:41Z" name: isvc-sklearn-scale-raw-dc419-predictor.18c103b4961e5f5f namespace: kserve-ci-e2e-test resourceVersion: "18397" uid: 10ef0f6e-3f7c-4a22-b6fd-9abf40df5753 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:16:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-dc419-predictor namespace: kserve-ci-e2e-test resourceVersion: "18392" uid: fc5adaac-2af0-410d-8fa6-999f3f02a81e kind: Event lastTimestamp: "2026-07-10T19:17:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:17:11Z" name: isvc-sklearn-scale-raw-dc419-predictor.18c103b814f95503 namespace: kserve-ci-e2e-test resourceVersion: "18690" uid: e30b3624-ef9e-4a96-8846-592c0bfa5156 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:16:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-dc419-predictor namespace: kserve-ci-e2e-test resourceVersion: "18392" uid: fc5adaac-2af0-410d-8fa6-999f3f02a81e kind: Event lastTimestamp: "2026-07-10T19:17:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:16:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:17:11Z" name: isvc-sklearn-scale-raw-dc419-predictor.18c103b8155d7bf8 namespace: kserve-ci-e2e-test resourceVersion: "18691" uid: 30278c19-3b69-4db1-a12d-6b51d31e2286 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-10T19:17:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-dc419-predictor namespace: kserve-ci-e2e-test resourceVersion: "18583" uid: fc5adaac-2af0-410d-8fa6-999f3f02a81e kind: Event lastTimestamp: "2026-07-10T19:19:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:17:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:19:11Z" name: isvc-sklearn-scale-raw-dc419-predictor.18c103bf13459162 namespace: kserve-ci-e2e-test resourceVersion: "19531" uid: 4a55e614-e94c-4295-b9dc-106e2a1a928b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-10T19:17:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-dc419-predictor namespace: kserve-ci-e2e-test resourceVersion: "18583" uid: fc5adaac-2af0-410d-8fa6-999f3f02a81e kind: Event lastTimestamp: "2026-07-10T19:19:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:17:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:19:11Z" name: isvc-sklearn-scale-raw-dc419-predictor.18c103bf13ae7f72 namespace: kserve-ci-e2e-test resourceVersion: "19532" uid: 95c1c506-24ef-4c06-aeb8-2473c0ce8647 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-10T19:16:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-dc419 namespace: kserve-ci-e2e-test resourceVersion: "18372" uid: 736754ee-cc30-4c67-bea9-f91bbbd3aafe kind: Event lastTimestamp: "2026-07-10T19:19:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:16:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:19:19Z" name: isvc-sklearn-scale-raw-dc419.18c103b49621ed90 namespace: kserve-ci-e2e-test resourceVersion: "19593" uid: c9ea4c91-4f97-4a89-893e-a810d915bc3e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:16:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-dc419 namespace: kserve-ci-e2e-test resourceVersion: "18372" uid: 736754ee-cc30-4c67-bea9-f91bbbd3aafe kind: Event lastTimestamp: "2026-07-10T19:16:41Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-dc419": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-dc419": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:16:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:16:41Z" name: isvc-sklearn-scale-raw-dc419.18c103b4967707e0 namespace: kserve-ci-e2e-test resourceVersion: "18439" uid: 9391d478-2a9a-417b-a14c-e0cb8ac61120 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:16:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-dc419 namespace: kserve-ci-e2e-test resourceVersion: "18391" uid: 736754ee-cc30-4c67-bea9-f91bbbd3aafe kind: Event lastTimestamp: "2026-07-10T19:16:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-dc419": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:16:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:16:41Z" name: isvc-sklearn-scale-raw-dc419.18c103b49e061584 namespace: kserve-ci-e2e-test resourceVersion: "18440" uid: a985d837-5592-4940-b1ad-527f20569ca5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-dc419 namespace: kserve-ci-e2e-test resourceVersion: "19590" uid: 736754ee-cc30-4c67-bea9-f91bbbd3aafe kind: Event lastTimestamp: "2026-07-10T19:19:19Z" message: InferenceService [isvc-sklearn-scale-raw-dc419] is Ready metadata: creationTimestamp: "2026-07-10T19:19:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:19:19Z" name: isvc-sklearn-scale-raw-dc419.18c103d95c743346 namespace: kserve-ci-e2e-test resourceVersion: "19592" uid: f8e6a9c1-14b2-4d64-b498-c5d95414bf3f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:11:15.795423Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: c773b2eb-474a-4cd6-98e2-5125175eb3f1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb to ip-10-0-134-26.ec2.internal metadata: creationTimestamp: "2026-07-10T19:11:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:11:15Z" name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb.18c10368bcdbd00a namespace: kserve-ci-e2e-test resourceVersion: "15440" uid: 9e9ed1fa-d25a-4034-8259-f0bec648e0e8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c9dd4b6f4-9plfp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:15Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: c773b2eb-474a-4cd6-98e2-5125175eb3f1 kind: Event lastTimestamp: "2026-07-10T19:11:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-e3394-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:15Z" name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb.18c10368c4efb3b7 namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: 747b2a8b-8da5-4d88-a563-62426da8ac69 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:16Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb namespace: kserve-ci-e2e-test resourceVersion: "15447" uid: c773b2eb-474a-4cd6-98e2-5125175eb3f1 kind: Event lastTimestamp: "2026-07-10T19:11:16Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:11:16Z" name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb.18c10368fa5ad7ba namespace: kserve-ci-e2e-test resourceVersion: "15475" uid: 6c5092c7-8439-479e-bdee-ae7131fcfcfd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: c773b2eb-474a-4cd6-98e2-5125175eb3f1 kind: Event lastTimestamp: "2026-07-10T19:11:16Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7893a590791cc9f33e3a3207dcc1d0843778bbf1aee8b6f22a63e3aa3b91a2e1" already present on machine metadata: creationTimestamp: "2026-07-10T19:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:16Z" name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb.18c10368fbd98926 namespace: kserve-ci-e2e-test resourceVersion: "15478" uid: 273e2ade-d49b-4ca4-b3e6-88e523c437ac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: c773b2eb-474a-4cd6-98e2-5125175eb3f1 kind: Event lastTimestamp: "2026-07-10T19:11:16Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:16Z" name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb.18c10368fede46c2 namespace: kserve-ci-e2e-test resourceVersion: "15480" uid: 76414e97-59d2-4276-b119-977bc29ffa4a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: c773b2eb-474a-4cd6-98e2-5125175eb3f1 kind: Event lastTimestamp: "2026-07-10T19:11:16Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:11:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:16Z" name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb.18c10368ff42b46a namespace: kserve-ci-e2e-test resourceVersion: "15481" uid: 827d2191-698c-4fc3-bc2b-f55bff114c52 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: c773b2eb-474a-4cd6-98e2-5125175eb3f1 kind: Event lastTimestamp: "2026-07-10T19:11:21Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-10T19:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:21Z" name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb.18c1036a0936d5ac namespace: kserve-ci-e2e-test resourceVersion: "15531" uid: cd29607f-814b-4069-94b8-bede5feb85d1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: c773b2eb-474a-4cd6-98e2-5125175eb3f1 kind: Event lastTimestamp: "2026-07-10T19:11:43Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 22.511s (22.511s including waiting). Image size: 1305567991 bytes.' metadata: creationTimestamp: "2026-07-10T19:11:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:43Z" name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb.18c1036f47042d6d namespace: kserve-ci-e2e-test resourceVersion: "15703" uid: 6b308729-2bb7-489c-a10d-8591fa5bd5e5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: c773b2eb-474a-4cd6-98e2-5125175eb3f1 kind: Event lastTimestamp: "2026-07-10T19:11:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:11:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:43Z" name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb.18c1036f4a4d5f5b namespace: kserve-ci-e2e-test resourceVersion: "15705" uid: 37232aec-83f2-49e5-b16b-6fcacd2b9dd1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: c773b2eb-474a-4cd6-98e2-5125175eb3f1 kind: Event lastTimestamp: "2026-07-10T19:11:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:11:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:43Z" name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb.18c1036f4ab010b8 namespace: kserve-ci-e2e-test resourceVersion: "15706" uid: 08824e2b-8077-4d35-ab72-14bd0a120ee6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: c773b2eb-474a-4cd6-98e2-5125175eb3f1 kind: Event lastTimestamp: "2026-07-10T19:11:43Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:11:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:43Z" name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb.18c1036f4abae1f8 namespace: kserve-ci-e2e-test resourceVersion: "15707" uid: 59fa5973-b5d9-4c4d-a8c6-ae237cb5e834 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: c773b2eb-474a-4cd6-98e2-5125175eb3f1 kind: Event lastTimestamp: "2026-07-10T19:11:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:11:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:43Z" name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb.18c1036f4ddf9a80 namespace: kserve-ci-e2e-test resourceVersion: "15709" uid: e6d6a6f8-f035-43e2-9143-bbc981119c44 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: c773b2eb-474a-4cd6-98e2-5125175eb3f1 kind: Event lastTimestamp: "2026-07-10T19:11:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:11:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:44Z" name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb.18c1036f4e42ca1f namespace: kserve-ci-e2e-test resourceVersion: "15710" uid: 69155d16-7da6-4127-9150-4bf42e1e8c36 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-10T19:11:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: c773b2eb-474a-4cd6-98e2-5125175eb3f1 kind: Event lastTimestamp: "2026-07-10T19:12:41Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:11:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:12:41Z" name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb.18c1036fa67365dd namespace: kserve-ci-e2e-test resourceVersion: "16144" uid: e1cbf7cf-b047-4a9a-8cb7-b17667cbfd05 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: c773b2eb-474a-4cd6-98e2-5125175eb3f1 kind: Event lastTimestamp: "2026-07-10T19:13:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:06Z" name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb.18c1038274ed286f namespace: kserve-ci-e2e-test resourceVersion: "16492" uid: d03b67f8-68ed-45bc-99c3-02d6e95a419d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: c773b2eb-474a-4cd6-98e2-5125175eb3f1 kind: Event lastTimestamp: "2026-07-10T19:13:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:06Z" name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb.18c1038274ed8591 namespace: kserve-ci-e2e-test resourceVersion: "16494" uid: 0664007a-4363-432b-a86e-085f5fcb7f76 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: c773b2eb-474a-4cd6-98e2-5125175eb3f1 kind: Event lastTimestamp: "2026-07-10T19:13:06Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:06Z" name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb.18c1038282c77fff namespace: kserve-ci-e2e-test resourceVersion: "16501" uid: 24276b64-8292-43f5-8f0f-a23d2122b577 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957 namespace: kserve-ci-e2e-test resourceVersion: "15423" uid: 71e1b21c-05b8-4880-a3c1-71b11411e1ff kind: Event lastTimestamp: "2026-07-10T19:11:15Z" message: 'Created pod: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957-9vgzb' metadata: creationTimestamp: "2026-07-10T19:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:15Z" name: isvc-xgboost-graph-raw-e3394-predictor-9c59fc957.18c10368bc7197e2 namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: 437f4b9e-e7b0-4b96-a2d3-faf09a109d84 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-e3394-predictor namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 3831fc81-23c3-4d96-83b0-27ddac806a90 kind: Event lastTimestamp: "2026-07-10T19:11:15Z" message: Scaled up replica set isvc-xgboost-graph-raw-e3394-predictor-9c59fc957 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:15Z" name: isvc-xgboost-graph-raw-e3394-predictor.18c10368b95a5e8a namespace: kserve-ci-e2e-test resourceVersion: "15425" uid: cef6eb55-28f3-4852-9134-a798f26c72cc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:11:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-e3394-predictor namespace: kserve-ci-e2e-test resourceVersion: "15422" uid: 5ed18e3c-241c-45ec-9852-39053abe3f88 kind: Event lastTimestamp: "2026-07-10T19:12:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:11:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:12:15Z" name: isvc-xgboost-graph-raw-e3394-predictor.18c1036c37ce172a namespace: kserve-ci-e2e-test resourceVersion: "15945" uid: 8221ca2c-8390-4f59-8003-806d542caa4c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:11:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-e3394-predictor namespace: kserve-ci-e2e-test resourceVersion: "15422" uid: 5ed18e3c-241c-45ec-9852-39053abe3f88 kind: Event lastTimestamp: "2026-07-10T19:12:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:11:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:12:15Z" name: isvc-xgboost-graph-raw-e3394-predictor.18c1036c38387b08 namespace: kserve-ci-e2e-test resourceVersion: "15946" uid: 7a042255-48b2-4d31-93bd-4c2fbac02730 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:12:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-e3394-predictor namespace: kserve-ci-e2e-test resourceVersion: "15607" uid: 5ed18e3c-241c-45ec-9852-39053abe3f88 kind: Event lastTimestamp: "2026-07-10T19:13:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:12:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:13:00Z" name: isvc-xgboost-graph-raw-e3394-predictor.18c1037a3262ed34 namespace: kserve-ci-e2e-test resourceVersion: "16327" uid: 1952ee86-5662-4a5f-a1c6-90987578606e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:12:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-e3394-predictor namespace: kserve-ci-e2e-test resourceVersion: "15607" uid: 5ed18e3c-241c-45ec-9852-39053abe3f88 kind: Event lastTimestamp: "2026-07-10T19:13:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:12:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:13:00Z" name: isvc-xgboost-graph-raw-e3394-predictor.18c1037a32d2612a namespace: kserve-ci-e2e-test resourceVersion: "16328" uid: 6cb0f59c-7ebb-4be2-8c01-c6a301d4619e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-10T19:11:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-e3394 namespace: kserve-ci-e2e-test resourceVersion: "15405" uid: f7f1f79d-bd65-4ac8-863d-3cc0f75bcb3d kind: Event lastTimestamp: "2026-07-10T19:12:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:12:51Z" name: isvc-xgboost-graph-raw-e3394.18c10368b95d6b00 namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: 63565756-566d-4037-9ffc-72adbf448e21 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-e3394 namespace: kserve-ci-e2e-test resourceVersion: "15405" uid: f7f1f79d-bd65-4ac8-863d-3cc0f75bcb3d kind: Event lastTimestamp: "2026-07-10T19:11:15Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-e3394": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-e3394": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:11:15Z" name: isvc-xgboost-graph-raw-e3394.18c10368ba216614 namespace: kserve-ci-e2e-test resourceVersion: "15431" uid: aa8c8bb8-16d0-4760-b788-03b6d1650b60 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:12:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-e3394 namespace: kserve-ci-e2e-test resourceVersion: "16222" uid: f7f1f79d-bd65-4ac8-863d-3cc0f75bcb3d kind: Event lastTimestamp: "2026-07-10T19:12:51Z" message: InferenceService [isvc-xgboost-graph-raw-e3394] is Ready metadata: creationTimestamp: "2026-07-10T19:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:12:51Z" name: isvc-xgboost-graph-raw-e3394.18c1037f0a8dc5a7 namespace: kserve-ci-e2e-test resourceVersion: "16223" uid: fea17789-ea9b-4f21-b79d-7e6cce6827a0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:13:06.134764Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb namespace: kserve-ci-e2e-test resourceVersion: "16456" uid: 4ae4fa15-4c11-4944-a7c0-f17bac5effaa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb to ip-10-0-134-26.ec2.internal metadata: creationTimestamp: "2026-07-10T19:13:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:13:06Z" name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb.18c103826d988f3d namespace: kserve-ci-e2e-test resourceVersion: "16462" uid: 1022088d-db66-426e-ac56-4e74c575b7fd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c9dd4b6f4-9plfp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:06Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb namespace: kserve-ci-e2e-test resourceVersion: "16471" uid: 4ae4fa15-4c11-4944-a7c0-f17bac5effaa kind: Event lastTimestamp: "2026-07-10T19:13:06Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:13:06Z" name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb.18c1038287e0ddfa namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 59165274-63e5-410d-a506-8e71d428b5d9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: 4ae4fa15-4c11-4944-a7c0-f17bac5effaa kind: Event lastTimestamp: "2026-07-10T19:13:06Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7893a590791cc9f33e3a3207dcc1d0843778bbf1aee8b6f22a63e3aa3b91a2e1" already present on machine metadata: creationTimestamp: "2026-07-10T19:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:06Z" name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb.18c10382893e014e namespace: kserve-ci-e2e-test resourceVersion: "16507" uid: 9a6cb3e7-3372-4e0a-ae38-e4055b5fde21 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: 4ae4fa15-4c11-4944-a7c0-f17bac5effaa kind: Event lastTimestamp: "2026-07-10T19:13:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:06Z" name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb.18c103828b9b49a3 namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: 80e5c849-32dc-4765-b155-58db481be550 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: 4ae4fa15-4c11-4944-a7c0-f17bac5effaa kind: Event lastTimestamp: "2026-07-10T19:13:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:06Z" name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb.18c103828c029811 namespace: kserve-ci-e2e-test resourceVersion: "16509" uid: 9e899f63-e6d1-4599-bc2b-6da5fecd2225 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: 4ae4fa15-4c11-4944-a7c0-f17bac5effaa kind: Event lastTimestamp: "2026-07-10T19:13:10Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-07-10T19:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:10Z" name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb.18c1038387469bb5 namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: 31ad84bc-634f-4eba-b4a0-ce6fa7b03832 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: 4ae4fa15-4c11-4944-a7c0-f17bac5effaa kind: Event lastTimestamp: "2026-07-10T19:13:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:10Z" name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb.18c103838f754fd5 namespace: kserve-ci-e2e-test resourceVersion: "16567" uid: d46dcf63-ba39-45f8-a865-c6f982201d20 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: 4ae4fa15-4c11-4944-a7c0-f17bac5effaa kind: Event lastTimestamp: "2026-07-10T19:13:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:11Z" name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb.18c103838fe28d6d namespace: kserve-ci-e2e-test resourceVersion: "16571" uid: a01f3be0-7256-47e9-802a-e3d7a123d6a2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: 4ae4fa15-4c11-4944-a7c0-f17bac5effaa kind: Event lastTimestamp: "2026-07-10T19:13:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:11Z" name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb.18c103838fed66ec namespace: kserve-ci-e2e-test resourceVersion: "16572" uid: ea0619dc-8603-4862-8095-ac22bfcb6c9d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: 4ae4fa15-4c11-4944-a7c0-f17bac5effaa kind: Event lastTimestamp: "2026-07-10T19:13:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:11Z" name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb.18c1038393985815 namespace: kserve-ci-e2e-test resourceVersion: "16576" uid: a0c11f2f-6784-4d88-9614-1f9ee061c215 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: 4ae4fa15-4c11-4944-a7c0-f17bac5effaa kind: Event lastTimestamp: "2026-07-10T19:13:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:11Z" name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb.18c10383942db2de namespace: kserve-ci-e2e-test resourceVersion: "16577" uid: ccd2066b-6eeb-45c1-a1fb-bffead3b81ba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-10T19:13:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: 4ae4fa15-4c11-4944-a7c0-f17bac5effaa kind: Event lastTimestamp: "2026-07-10T19:14:58Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:13:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:14:58Z" name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb.18c10383ffb1dedf namespace: kserve-ci-e2e-test resourceVersion: "17532" uid: 7fd9caae-26d0-4850-b515-c586e45fc9d6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: 4ae4fa15-4c11-4944-a7c0-f17bac5effaa kind: Event lastTimestamp: "2026-07-10T19:14:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:14:56Z" name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb.18c1039c19eee371 namespace: kserve-ci-e2e-test resourceVersion: "17494" uid: 4ed2cd92-3d44-4fef-9517-f1493648048c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: 4ae4fa15-4c11-4944-a7c0-f17bac5effaa kind: Event lastTimestamp: "2026-07-10T19:14:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:14:56Z" name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb.18c1039c19ef5cb7 namespace: kserve-ci-e2e-test resourceVersion: "17495" uid: da29ca93-62d2-4883-a721-55265cbf7904 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: 4ae4fa15-4c11-4944-a7c0-f17bac5effaa kind: Event lastTimestamp: "2026-07-10T19:14:58Z" message: 'Readiness probe failed: Get "https://10.133.0.28:8643/healthz": dial tcp 10.133.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:14:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:14:58Z" name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb.18c1039cadfc2a01 namespace: kserve-ci-e2e-test resourceVersion: "17531" uid: 2b2785a5-b4c3-44dd-a585-952523674625 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449 namespace: kserve-ci-e2e-test resourceVersion: "16446" uid: 29c5f060-54c5-4066-b0d4-a4a095561bb6 kind: Event lastTimestamp: "2026-07-10T19:13:06Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449-znpdb' metadata: creationTimestamp: "2026-07-10T19:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:13:06Z" name: isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449.18c103826cf4c771 namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: 7de5db58-6827-42d9-beda-3ab4fd925901 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-fed9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16443" uid: 19342790-e8a5-4eb5-8038-fc70d8d348ce kind: Event lastTimestamp: "2026-07-10T19:13:06Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-fed9d-predictor-55bbcc449 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:13:06Z" name: isvc-xgboost-graph-raw-hpa-fed9d-predictor.18c1038269d628c6 namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: f8d1e2ac-ccd2-4a49-8a61-6464a7f6b559 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:13:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-fed9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16445" uid: 3312b0a4-8946-47fb-becf-46f5ee29c224 kind: Event lastTimestamp: "2026-07-10T19:14:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:14:06Z" name: isvc-xgboost-graph-raw-hpa-fed9d-predictor.18c10385e8400bd7 namespace: kserve-ci-e2e-test resourceVersion: "16992" uid: 71f9266c-ea1f-4e23-9783-c61c16c87b85 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:13:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-fed9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16445" uid: 3312b0a4-8946-47fb-becf-46f5ee29c224 kind: Event lastTimestamp: "2026-07-10T19:14:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:14:06Z" name: isvc-xgboost-graph-raw-hpa-fed9d-predictor.18c10385e8a6fa5f namespace: kserve-ci-e2e-test resourceVersion: "16993" uid: 41234b23-d592-4d20-bbf6-44fbf9bf4acd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:14:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-fed9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16671" uid: 3312b0a4-8946-47fb-becf-46f5ee29c224 kind: Event lastTimestamp: "2026-07-10T19:14:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:14:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:14:51Z" name: isvc-xgboost-graph-raw-hpa-fed9d-predictor.18c10393e3423d94 namespace: kserve-ci-e2e-test resourceVersion: "17370" uid: 65e6ca54-c0c5-4f66-b14b-0d6cbad944ae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:14:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-fed9d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16671" uid: 3312b0a4-8946-47fb-becf-46f5ee29c224 kind: Event lastTimestamp: "2026-07-10T19:14:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:14:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:14:51Z" name: isvc-xgboost-graph-raw-hpa-fed9d-predictor.18c10393e3a3282c namespace: kserve-ci-e2e-test resourceVersion: "17371" uid: 9a7661d1-5f63-4f76-b31c-722f84eb2659 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-10T19:13:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-fed9d namespace: kserve-ci-e2e-test resourceVersion: "16426" uid: 09eeeb90-1885-4d4b-b68d-ad7daa60e2ea kind: Event lastTimestamp: "2026-07-10T19:14:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:14:19Z" name: isvc-xgboost-graph-raw-hpa-fed9d.18c103826a07f559 namespace: kserve-ci-e2e-test resourceVersion: "17096" uid: d2b50f0a-dd5f-4b62-a8af-d4c7dd7b663e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-fed9d namespace: kserve-ci-e2e-test resourceVersion: "17093" uid: 09eeeb90-1885-4d4b-b68d-ad7daa60e2ea kind: Event lastTimestamp: "2026-07-10T19:14:18Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-fed9d] is Ready metadata: creationTimestamp: "2026-07-10T19:14:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:14:18Z" name: isvc-xgboost-graph-raw-hpa-fed9d.18c1039363066f25 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 75e0e454-e89e-4020-8685-3a3ac947e3df reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:14:56.281131Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-358e7-predictor-9f5c55544-9gxsq namespace: kserve-ci-e2e-test resourceVersion: "17459" uid: deb26912-ed75-416d-814c-1de8d6d5f0c1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-358e7-predictor-9f5c55544-9gxsq to ip-10-0-134-26.ec2.internal metadata: creationTimestamp: "2026-07-10T19:14:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:14:56Z" name: message-dumper-raw-358e7-predictor-9f5c55544-9gxsq.18c1039c12d4a702 namespace: kserve-ci-e2e-test resourceVersion: "17466" uid: 17a29015-c3f7-4b4c-bdcf-81bee56ddfe0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c9dd4b6f4-9plfp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:56Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-358e7-predictor-9f5c55544-9gxsq namespace: kserve-ci-e2e-test resourceVersion: "17473" uid: deb26912-ed75-416d-814c-1de8d6d5f0c1 kind: Event lastTimestamp: "2026-07-10T19:14:56Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:14:56Z" name: message-dumper-raw-358e7-predictor-9f5c55544-9gxsq.18c1039c2c9adccf namespace: kserve-ci-e2e-test resourceVersion: "17502" uid: 48d7c66c-6544-4809-bf37-b22fd3452277 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-358e7-predictor-9f5c55544-9gxsq namespace: kserve-ci-e2e-test resourceVersion: "17462" uid: deb26912-ed75-416d-814c-1de8d6d5f0c1 kind: Event lastTimestamp: "2026-07-10T19:14:56Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-10T19:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:14:56Z" name: message-dumper-raw-358e7-predictor-9f5c55544-9gxsq.18c1039c2de1bbe9 namespace: kserve-ci-e2e-test resourceVersion: "17504" uid: d40b91d0-06e2-48e2-8abd-7223ea8c8fb4 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-358e7-predictor-9f5c55544-9gxsq namespace: kserve-ci-e2e-test resourceVersion: "17462" uid: deb26912-ed75-416d-814c-1de8d6d5f0c1 kind: Event lastTimestamp: "2026-07-10T19:14:57Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.059s (1.059s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-10T19:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:14:57Z" name: message-dumper-raw-358e7-predictor-9f5c55544-9gxsq.18c1039c6d020edb namespace: kserve-ci-e2e-test resourceVersion: "17513" uid: 74d8fb88-4e0a-493a-8425-0bf2efa723c2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-358e7-predictor-9f5c55544-9gxsq namespace: kserve-ci-e2e-test resourceVersion: "17462" uid: deb26912-ed75-416d-814c-1de8d6d5f0c1 kind: Event lastTimestamp: "2026-07-10T19:14:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:14:57Z" name: message-dumper-raw-358e7-predictor-9f5c55544-9gxsq.18c1039c6ff81de5 namespace: kserve-ci-e2e-test resourceVersion: "17516" uid: 0f1ec39f-4f63-491f-81ea-0f0b2e6f9575 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-358e7-predictor-9f5c55544-9gxsq namespace: kserve-ci-e2e-test resourceVersion: "17462" uid: deb26912-ed75-416d-814c-1de8d6d5f0c1 kind: Event lastTimestamp: "2026-07-10T19:14:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:14:57Z" name: message-dumper-raw-358e7-predictor-9f5c55544-9gxsq.18c1039c70883997 namespace: kserve-ci-e2e-test resourceVersion: "17517" uid: b3023487-7022-420d-98c0-658c50a2da20 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-358e7-predictor-9f5c55544-9gxsq namespace: kserve-ci-e2e-test resourceVersion: "17462" uid: deb26912-ed75-416d-814c-1de8d6d5f0c1 kind: Event lastTimestamp: "2026-07-10T19:14:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:14:57Z" name: message-dumper-raw-358e7-predictor-9f5c55544-9gxsq.18c1039c70939e3c namespace: kserve-ci-e2e-test resourceVersion: "17518" uid: bdc64fa3-d69f-49b7-a139-d195a6948cad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-358e7-predictor-9f5c55544-9gxsq namespace: kserve-ci-e2e-test resourceVersion: "17462" uid: deb26912-ed75-416d-814c-1de8d6d5f0c1 kind: Event lastTimestamp: "2026-07-10T19:14:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:14:57Z" name: message-dumper-raw-358e7-predictor-9f5c55544-9gxsq.18c1039c736a37e9 namespace: kserve-ci-e2e-test resourceVersion: "17519" uid: 9874684f-c06a-42b2-b376-c5818c8ad29b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-358e7-predictor-9f5c55544-9gxsq namespace: kserve-ci-e2e-test resourceVersion: "17462" uid: deb26912-ed75-416d-814c-1de8d6d5f0c1 kind: Event lastTimestamp: "2026-07-10T19:14:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:14:57Z" name: message-dumper-raw-358e7-predictor-9f5c55544-9gxsq.18c1039c73dc964f namespace: kserve-ci-e2e-test resourceVersion: "17521" uid: 0962cc47-25ef-46d7-bc50-81a396411c73 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:16:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-358e7-predictor-9f5c55544-9gxsq namespace: kserve-ci-e2e-test resourceVersion: "17462" uid: deb26912-ed75-416d-814c-1de8d6d5f0c1 kind: Event lastTimestamp: "2026-07-10T19:16:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:16:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:16:41Z" name: message-dumper-raw-358e7-predictor-9f5c55544-9gxsq.18c103b49f149557 namespace: kserve-ci-e2e-test resourceVersion: "18442" uid: 2531a952-9431-4668-a86f-cb6867717bed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:16:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-358e7-predictor-9f5c55544-9gxsq namespace: kserve-ci-e2e-test resourceVersion: "17462" uid: deb26912-ed75-416d-814c-1de8d6d5f0c1 kind: Event lastTimestamp: "2026-07-10T19:16:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:16:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:16:41Z" name: message-dumper-raw-358e7-predictor-9f5c55544-9gxsq.18c103b49f151247 namespace: kserve-ci-e2e-test resourceVersion: "18443" uid: 0d62d639-06f5-4675-b57f-ab62ec79784d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-358e7-predictor-9f5c55544 namespace: kserve-ci-e2e-test resourceVersion: "17441" uid: a313516b-fc9e-4da9-ac1a-f3aaa65901fb kind: Event lastTimestamp: "2026-07-10T19:14:56Z" message: 'Created pod: message-dumper-raw-358e7-predictor-9f5c55544-9gxsq' metadata: creationTimestamp: "2026-07-10T19:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:14:56Z" name: message-dumper-raw-358e7-predictor-9f5c55544.18c1039c123d1d17 namespace: kserve-ci-e2e-test resourceVersion: "17461" uid: 000c7eb9-89a6-4adb-9a48-51233feda950 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-358e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17434" uid: 8975c560-905b-44ab-a08c-24eb4ea0e38c kind: Event lastTimestamp: "2026-07-10T19:14:56Z" message: Scaled up replica set message-dumper-raw-358e7-predictor-9f5c55544 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:14:56Z" name: message-dumper-raw-358e7-predictor.18c1039c0e816024 namespace: kserve-ci-e2e-test resourceVersion: "17445" uid: 0c165f45-2a18-4054-aa06-36ffe91aa6da reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:15:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-358e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17436" uid: c95d5a13-9f2c-408b-9b4b-58b1bceda1bb kind: Event lastTimestamp: "2026-07-10T19:15:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:15:26Z" name: message-dumper-raw-358e7-predictor.18c1039f8cc0b716 namespace: kserve-ci-e2e-test resourceVersion: "17815" uid: b9fc481d-f639-433a-a1d8-85c64b6d9137 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:15:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-358e7-predictor namespace: kserve-ci-e2e-test resourceVersion: "17436" uid: c95d5a13-9f2c-408b-9b4b-58b1bceda1bb kind: Event lastTimestamp: "2026-07-10T19:15:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:15:26Z" name: message-dumper-raw-358e7-predictor.18c1039f8d1ea91f namespace: kserve-ci-e2e-test resourceVersion: "17816" uid: e98e75a8-2484-4c0e-b4f9-3a5914bba95e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-10T19:14:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-358e7 namespace: kserve-ci-e2e-test resourceVersion: "17428" uid: 5d561810-a622-42d1-bfbc-723dc64bce5f kind: Event lastTimestamp: "2026-07-10T19:15:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:15:05Z" name: message-dumper-raw-358e7.18c1039c0e41c073 namespace: kserve-ci-e2e-test resourceVersion: "17598" uid: b29c34f9-4f71-4852-91dc-153b9c2a6bf2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-358e7 namespace: kserve-ci-e2e-test resourceVersion: "17443" uid: 5d561810-a622-42d1-bfbc-723dc64bce5f kind: Event lastTimestamp: "2026-07-10T19:14:56Z" message: 'Failed to update status for InferenceService "message-dumper-raw-358e7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-358e7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:14:56Z" name: message-dumper-raw-358e7.18c1039c12dacc60 namespace: kserve-ci-e2e-test resourceVersion: "17467" uid: c61e6e8a-13aa-424b-b3ef-a970c1938cfe reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-358e7 namespace: kserve-ci-e2e-test resourceVersion: "17596" uid: 5d561810-a622-42d1-bfbc-723dc64bce5f kind: Event lastTimestamp: "2026-07-10T19:15:05Z" message: InferenceService [message-dumper-raw-358e7] is Ready metadata: creationTimestamp: "2026-07-10T19:15:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:15:05Z" name: message-dumper-raw-358e7.18c1039e33689a90 namespace: kserve-ci-e2e-test resourceVersion: "17597" uid: 675d80fe-fd94-4efb-9d5e-67de625ec80f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:12:55.665319Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-e3394-5c795895db-9tdkc namespace: kserve-ci-e2e-test resourceVersion: "16269" uid: 2a112353-8b4b-4343-b702-6c60ac82c551 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-e3394-5c795895db-9tdkc to ip-10-0-134-26.ec2.internal metadata: creationTimestamp: "2026-07-10T19:12:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:12:55Z" name: model-chainer-raw-e3394-5c795895db-9tdkc.18c1037ffd91ba2e namespace: kserve-ci-e2e-test resourceVersion: "16274" uid: 93e24862-8fbd-464d-85d3-ecc240e9cc7c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c9dd4b6f4-9plfp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:12:56Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-e3394-5c795895db-9tdkc namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 2a112353-8b4b-4343-b702-6c60ac82c551 kind: Event lastTimestamp: "2026-07-10T19:12:56Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:12:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:12:56Z" name: model-chainer-raw-e3394-5c795895db-9tdkc.18c10380173e3dd7 namespace: kserve-ci-e2e-test resourceVersion: "16287" uid: 7fbd3d5e-9b1a-4c71-9165-a2e3043f0472 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:12:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-e3394} kind: Pod name: model-chainer-raw-e3394-5c795895db-9tdkc namespace: kserve-ci-e2e-test resourceVersion: "16272" uid: 2a112353-8b4b-4343-b702-6c60ac82c551 kind: Event lastTimestamp: "2026-07-10T19:12:56Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:73feeff3bf9ed8a8540674263cab5a50d6f54742b256a955c4797c06bcd7349c" metadata: creationTimestamp: "2026-07-10T19:12:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:12:56Z" name: model-chainer-raw-e3394-5c795895db-9tdkc.18c10380188150db namespace: kserve-ci-e2e-test resourceVersion: "16289" uid: 814bfc88-170e-44b2-ad18-192449904a10 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:12:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-e3394} kind: Pod name: model-chainer-raw-e3394-5c795895db-9tdkc namespace: kserve-ci-e2e-test resourceVersion: "16272" uid: 2a112353-8b4b-4343-b702-6c60ac82c551 kind: Event lastTimestamp: "2026-07-10T19:12:58Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:73feeff3bf9ed8a8540674263cab5a50d6f54742b256a955c4797c06bcd7349c" in 2.23s (2.23s including waiting). Image size: 218072665 bytes.' metadata: creationTimestamp: "2026-07-10T19:12:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:12:58Z" name: model-chainer-raw-e3394-5c795895db-9tdkc.18c103809d6e76a6 namespace: kserve-ci-e2e-test resourceVersion: "16304" uid: eb508736-d2fb-4454-aa20-7457a4576e58 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:12:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-e3394} kind: Pod name: model-chainer-raw-e3394-5c795895db-9tdkc namespace: kserve-ci-e2e-test resourceVersion: "16272" uid: 2a112353-8b4b-4343-b702-6c60ac82c551 kind: Event lastTimestamp: "2026-07-10T19:12:58Z" message: 'Created container: model-chainer-raw-e3394' metadata: creationTimestamp: "2026-07-10T19:12:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:12:58Z" name: model-chainer-raw-e3394-5c795895db-9tdkc.18c10380a0d7b1b9 namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: ba122311-8568-41fc-afc7-ac7f86ca39da reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:12:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-e3394} kind: Pod name: model-chainer-raw-e3394-5c795895db-9tdkc namespace: kserve-ci-e2e-test resourceVersion: "16272" uid: 2a112353-8b4b-4343-b702-6c60ac82c551 kind: Event lastTimestamp: "2026-07-10T19:12:58Z" message: Started container model-chainer-raw-e3394 metadata: creationTimestamp: "2026-07-10T19:12:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:12:58Z" name: model-chainer-raw-e3394-5c795895db-9tdkc.18c10380a13de12b namespace: kserve-ci-e2e-test resourceVersion: "16306" uid: bfc863e3-4416-42e4-9ead-02eab65592f1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-e3394} kind: Pod name: model-chainer-raw-e3394-5c795895db-9tdkc namespace: kserve-ci-e2e-test resourceVersion: "16272" uid: 2a112353-8b4b-4343-b702-6c60ac82c551 kind: Event lastTimestamp: "2026-07-10T19:13:05Z" message: Stopping container model-chainer-raw-e3394 metadata: creationTimestamp: "2026-07-10T19:13:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:05Z" name: model-chainer-raw-e3394-5c795895db-9tdkc.18c103825348912d namespace: kserve-ci-e2e-test resourceVersion: "16388" uid: 305d6648-cf34-4227-9c71-337c5e84cd45 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:13:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-e3394} kind: Pod name: model-chainer-raw-e3394-5c795895db-9tdkc namespace: kserve-ci-e2e-test resourceVersion: "16272" uid: 2a112353-8b4b-4343-b702-6c60ac82c551 kind: Event lastTimestamp: "2026-07-10T19:13:34Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-10T19:13:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:34Z" name: model-chainer-raw-e3394-5c795895db-9tdkc.18c10383481d00b1 namespace: kserve-ci-e2e-test resourceVersion: "16762" uid: 14f96e0f-af77-4cc0-abe0-ca902bde8986 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:12:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-e3394-5c795895db namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: 64c8c897-4a7c-4c3b-8386-84bec5b74af1 kind: Event lastTimestamp: "2026-07-10T19:12:55Z" message: 'Created pod: model-chainer-raw-e3394-5c795895db-9tdkc' metadata: creationTimestamp: "2026-07-10T19:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:12:55Z" name: model-chainer-raw-e3394-5c795895db.18c1037ffcf3fde3 namespace: kserve-ci-e2e-test resourceVersion: "16271" uid: a28a1bdd-5652-4845-9036-e607be532ad2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:12:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-e3394 namespace: kserve-ci-e2e-test resourceVersion: "16264" uid: 606f1fc8-8035-423a-ab3c-36dd50504535 kind: Event lastTimestamp: "2026-07-10T19:12:55Z" message: Scaled up replica set model-chainer-raw-e3394-5c795895db from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:12:55Z" name: model-chainer-raw-e3394.18c1037ffbd26350 namespace: kserve-ci-e2e-test resourceVersion: "16267" uid: bcd01acf-3e17-4871-81df-23de5bd7dd80 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:12:55Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-e3394 namespace: kserve-ci-e2e-test resourceVersion: "16282" uid: 1d9668ad-7a29-4723-92fb-6469fcffb234 kind: Event lastTimestamp: "2026-07-10T19:12:55Z" message: InferenceGraph [model-chainer-raw-e3394] is Ready metadata: creationTimestamp: "2026-07-10T19:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:12:55Z" name: model-chainer-raw-e3394.18c10380026db267 namespace: kserve-ci-e2e-test resourceVersion: "16284" uid: 954629b9-a4e6-405e-bd75-e9c2bbcd8191 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:14:45.987397Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-fed9d-65f5bbb578-5jmqk namespace: kserve-ci-e2e-test resourceVersion: "17311" uid: 9805083d-e472-4d19-a017-3bcc58721940 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-fed9d-65f5bbb578-5jmqk to ip-10-0-134-26.ec2.internal metadata: creationTimestamp: "2026-07-10T19:14:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:14:45Z" name: model-chainer-raw-hpa-fed9d-65f5bbb578-5jmqk.18c10399ad46c8ec namespace: kserve-ci-e2e-test resourceVersion: "17316" uid: 02c7bf60-c941-4cb1-a3ec-d5bbfb05ae39 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c9dd4b6f4-9plfp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:46Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-fed9d-65f5bbb578-5jmqk namespace: kserve-ci-e2e-test resourceVersion: "17314" uid: 9805083d-e472-4d19-a017-3bcc58721940 kind: Event lastTimestamp: "2026-07-10T19:14:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-hpa-fed9d-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:14:46Z" name: model-chainer-raw-hpa-fed9d-65f5bbb578-5jmqk.18c10399b7b9152e namespace: kserve-ci-e2e-test resourceVersion: "17326" uid: 73b40e9f-a705-47d8-a82f-57cfb94d58b1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:47Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-fed9d-65f5bbb578-5jmqk namespace: kserve-ci-e2e-test resourceVersion: "17320" uid: 9805083d-e472-4d19-a017-3bcc58721940 kind: Event lastTimestamp: "2026-07-10T19:14:47Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:14:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:14:47Z" name: model-chainer-raw-hpa-fed9d-65f5bbb578-5jmqk.18c10399ea666ce9 namespace: kserve-ci-e2e-test resourceVersion: "17333" uid: 7f3a8843-ae3a-4064-af22-c29f275bbb9e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-fed9d} kind: Pod name: model-chainer-raw-hpa-fed9d-65f5bbb578-5jmqk namespace: kserve-ci-e2e-test resourceVersion: "17314" uid: 9805083d-e472-4d19-a017-3bcc58721940 kind: Event lastTimestamp: "2026-07-10T19:14:47Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:73feeff3bf9ed8a8540674263cab5a50d6f54742b256a955c4797c06bcd7349c" already present on machine metadata: creationTimestamp: "2026-07-10T19:14:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:14:47Z" name: model-chainer-raw-hpa-fed9d-65f5bbb578-5jmqk.18c10399ebe03fdc namespace: kserve-ci-e2e-test resourceVersion: "17335" uid: c6a4bc56-a311-49fb-9e2b-d346e708be80 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-fed9d} kind: Pod name: model-chainer-raw-hpa-fed9d-65f5bbb578-5jmqk namespace: kserve-ci-e2e-test resourceVersion: "17314" uid: 9805083d-e472-4d19-a017-3bcc58721940 kind: Event lastTimestamp: "2026-07-10T19:14:47Z" message: 'Created container: model-chainer-raw-hpa-fed9d' metadata: creationTimestamp: "2026-07-10T19:14:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:14:47Z" name: model-chainer-raw-hpa-fed9d-65f5bbb578-5jmqk.18c10399eed891b1 namespace: kserve-ci-e2e-test resourceVersion: "17337" uid: d7ec1e67-dd98-4d8d-91e6-04ef7fe562b8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-fed9d} kind: Pod name: model-chainer-raw-hpa-fed9d-65f5bbb578-5jmqk namespace: kserve-ci-e2e-test resourceVersion: "17314" uid: 9805083d-e472-4d19-a017-3bcc58721940 kind: Event lastTimestamp: "2026-07-10T19:14:47Z" message: Started container model-chainer-raw-hpa-fed9d metadata: creationTimestamp: "2026-07-10T19:14:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:14:47Z" name: model-chainer-raw-hpa-fed9d-65f5bbb578-5jmqk.18c10399ef4f64da namespace: kserve-ci-e2e-test resourceVersion: "17338" uid: 6756e3f2-ee71-4552-9160-13cff662b791 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-fed9d} kind: Pod name: model-chainer-raw-hpa-fed9d-65f5bbb578-5jmqk namespace: kserve-ci-e2e-test resourceVersion: "17314" uid: 9805083d-e472-4d19-a017-3bcc58721940 kind: Event lastTimestamp: "2026-07-10T19:14:56Z" message: Stopping container model-chainer-raw-hpa-fed9d metadata: creationTimestamp: "2026-07-10T19:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:14:56Z" name: model-chainer-raw-hpa-fed9d-65f5bbb578-5jmqk.18c1039c05116e9a namespace: kserve-ci-e2e-test resourceVersion: "17423" uid: 31624563-6f6b-47a8-a5c9-12e7e5f3ea49 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:14:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-fed9d} kind: Pod name: model-chainer-raw-hpa-fed9d-65f5bbb578-5jmqk namespace: kserve-ci-e2e-test resourceVersion: "17314" uid: 9805083d-e472-4d19-a017-3bcc58721940 kind: Event lastTimestamp: "2026-07-10T19:15:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-10T19:14:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:15:23Z" name: model-chainer-raw-hpa-fed9d-65f5bbb578-5jmqk.18c1039c89e40216 namespace: kserve-ci-e2e-test resourceVersion: "17794" uid: d3c18920-0272-4aa2-918b-5ed8d17b92b7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-fed9d-65f5bbb578 namespace: kserve-ci-e2e-test resourceVersion: "17308" uid: 5dad8dc2-8559-49fd-bf4e-3388c25af25f kind: Event lastTimestamp: "2026-07-10T19:14:45Z" message: 'Created pod: model-chainer-raw-hpa-fed9d-65f5bbb578-5jmqk' metadata: creationTimestamp: "2026-07-10T19:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:14:45Z" name: model-chainer-raw-hpa-fed9d-65f5bbb578.18c10399acd6ccfc namespace: kserve-ci-e2e-test resourceVersion: "17313" uid: fe6fa953-5638-4ea4-bdce-b8471a0dd84a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-fed9d namespace: kserve-ci-e2e-test resourceVersion: "17306" uid: 11f5aad7-bc09-4215-aaed-70d242ae052b kind: Event lastTimestamp: "2026-07-10T19:14:45Z" message: Scaled up replica set model-chainer-raw-hpa-fed9d-65f5bbb578 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:14:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:14:45Z" name: model-chainer-raw-hpa-fed9d.18c10399ab8f3ac8 namespace: kserve-ci-e2e-test resourceVersion: "17309" uid: 7ed1dcb9-28d1-4cf0-8a77-e02c3ef27038 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:14:46Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-fed9d namespace: kserve-ci-e2e-test resourceVersion: "17322" uid: 73b0811e-b96e-48e3-b7ed-8cd5413e087d kind: Event lastTimestamp: "2026-07-10T19:14:46Z" message: InferenceGraph [model-chainer-raw-hpa-fed9d] is Ready metadata: creationTimestamp: "2026-07-10T19:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:14:46Z" name: model-chainer-raw-hpa-fed9d.18c10399b0f0c97d namespace: kserve-ci-e2e-test resourceVersion: "17324" uid: 7211ba2f-4e57-4288-9fc8-d41b92667e89 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:21:35.665700Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-e7135-predictor-69d867d788-dssrm namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: 5b207f5e-df39-431c-9909-47d282555c9b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-e7135-predictor-69d867d788-dssrm to ip-10-0-134-26.ec2.internal metadata: creationTimestamp: "2026-07-10T19:21:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:21:35Z" name: raw-sklearn-e7135-predictor-69d867d788-dssrm.18c103f91001a154 namespace: kserve-ci-e2e-test resourceVersion: "20865" uid: 58880019-2f5c-45bf-a5ba-3b3f95c76404 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c9dd4b6f4-9plfp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:36Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-e7135-predictor-69d867d788-dssrm namespace: kserve-ci-e2e-test resourceVersion: "20869" uid: 5b207f5e-df39-431c-9909-47d282555c9b kind: Event lastTimestamp: "2026-07-10T19:21:36Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:21:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:21:36Z" name: raw-sklearn-e7135-predictor-69d867d788-dssrm.18c103f92946ade7 namespace: kserve-ci-e2e-test resourceVersion: "20873" uid: fe46c04a-3198-4ead-932e-3375baee1115 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-e7135-predictor-69d867d788-dssrm namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: 5b207f5e-df39-431c-9909-47d282555c9b kind: Event lastTimestamp: "2026-07-10T19:21:36Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7893a590791cc9f33e3a3207dcc1d0843778bbf1aee8b6f22a63e3aa3b91a2e1" already present on machine metadata: creationTimestamp: "2026-07-10T19:21:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:36Z" name: raw-sklearn-e7135-predictor-69d867d788-dssrm.18c103f92aa98cb3 namespace: kserve-ci-e2e-test resourceVersion: "20875" uid: 6e44bfd8-48c6-43b7-a446-500c356df683 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-e7135-predictor-69d867d788-dssrm namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: 5b207f5e-df39-431c-9909-47d282555c9b kind: Event lastTimestamp: "2026-07-10T19:21:36Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:21:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:36Z" name: raw-sklearn-e7135-predictor-69d867d788-dssrm.18c103f92d1c507e namespace: kserve-ci-e2e-test resourceVersion: "20878" uid: 3bc73f05-d540-416f-a410-e170f9f2026a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-e7135-predictor-69d867d788-dssrm namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: 5b207f5e-df39-431c-9909-47d282555c9b kind: Event lastTimestamp: "2026-07-10T19:21:36Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:21:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:36Z" name: raw-sklearn-e7135-predictor-69d867d788-dssrm.18c103f92d8c3207 namespace: kserve-ci-e2e-test resourceVersion: "20879" uid: eb6dcc9c-b415-4f6e-9e26-92b12e972f0e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-e7135-predictor-69d867d788-dssrm namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: 5b207f5e-df39-431c-9909-47d282555c9b kind: Event lastTimestamp: "2026-07-10T19:21:39Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:21:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:39Z" name: raw-sklearn-e7135-predictor-69d867d788-dssrm.18c103fa11d769e6 namespace: kserve-ci-e2e-test resourceVersion: "20911" uid: 1231776e-1f0c-4e24-b183-61afd54dcb55 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-e7135-predictor-69d867d788-dssrm namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: 5b207f5e-df39-431c-9909-47d282555c9b kind: Event lastTimestamp: "2026-07-10T19:21:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:40Z" name: raw-sklearn-e7135-predictor-69d867d788-dssrm.18c103fa14f9bc28 namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: 11e1d11a-c010-4447-ad72-2ab99f06d85f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-e7135-predictor-69d867d788-dssrm namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: 5b207f5e-df39-431c-9909-47d282555c9b kind: Event lastTimestamp: "2026-07-10T19:21:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:40Z" name: raw-sklearn-e7135-predictor-69d867d788-dssrm.18c103fa156a2c57 namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 47c51928-4b6e-47f8-a5c5-aeca3f943d6b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-e7135-predictor-69d867d788-dssrm namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: 5b207f5e-df39-431c-9909-47d282555c9b kind: Event lastTimestamp: "2026-07-10T19:21:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:40Z" name: raw-sklearn-e7135-predictor-69d867d788-dssrm.18c103fa1574b22d namespace: kserve-ci-e2e-test resourceVersion: "20917" uid: 8f508250-c243-4aaa-a78d-e00651a678c6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-e7135-predictor-69d867d788-dssrm namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: 5b207f5e-df39-431c-9909-47d282555c9b kind: Event lastTimestamp: "2026-07-10T19:21:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:40Z" name: raw-sklearn-e7135-predictor-69d867d788-dssrm.18c103fa1866663f namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: bb97404d-f5a7-472b-9364-01b1e859eeeb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-e7135-predictor-69d867d788-dssrm namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: 5b207f5e-df39-431c-9909-47d282555c9b kind: Event lastTimestamp: "2026-07-10T19:21:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:21:40Z" name: raw-sklearn-e7135-predictor-69d867d788-dssrm.18c103fa18d31f45 namespace: kserve-ci-e2e-test resourceVersion: "20920" uid: 1f8506b9-f122-4243-bbd4-fb8930456a4c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-10T19:21:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-e7135-predictor-69d867d788-dssrm namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: 5b207f5e-df39-431c-9909-47d282555c9b kind: Event lastTimestamp: "2026-07-10T19:23:08Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:21:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:23:08Z" name: raw-sklearn-e7135-predictor-69d867d788-dssrm.18c103fa89a818ab namespace: kserve-ci-e2e-test resourceVersion: "21566" uid: dfa23574-cd20-4a39-ac08-f35e82ccfa7e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:23:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-e7135-predictor-69d867d788-dssrm namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: 5b207f5e-df39-431c-9909-47d282555c9b kind: Event lastTimestamp: "2026-07-10T19:23:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:23:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:23:05Z" name: raw-sklearn-e7135-predictor-69d867d788-dssrm.18c1040e0fd1ce29 namespace: kserve-ci-e2e-test resourceVersion: "21547" uid: 9a7d1afd-1eb5-4ecd-95fd-3950e5b52ce5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:23:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-e7135-predictor-69d867d788-dssrm namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: 5b207f5e-df39-431c-9909-47d282555c9b kind: Event lastTimestamp: "2026-07-10T19:23:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:23:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:23:05Z" name: raw-sklearn-e7135-predictor-69d867d788-dssrm.18c1040e0fd242a5 namespace: kserve-ci-e2e-test resourceVersion: "21548" uid: 237d5aa8-b3ed-4478-ac98-4689fdc26f13 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:23:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-e7135-predictor-69d867d788-dssrm namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: 5b207f5e-df39-431c-9909-47d282555c9b kind: Event lastTimestamp: "2026-07-10T19:23:08Z" message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": dial tcp 10.133.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:23:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:23:08Z" name: raw-sklearn-e7135-predictor-69d867d788-dssrm.18c1040e8fd3c16a namespace: kserve-ci-e2e-test resourceVersion: "21565" uid: 40023bda-9102-4661-96ac-5cf324689b00 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-e7135-predictor-69d867d788 namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: 175d4ed8-24f3-4892-92d7-74753e8353ec kind: Event lastTimestamp: "2026-07-10T19:21:35Z" message: 'Created pod: raw-sklearn-e7135-predictor-69d867d788-dssrm' metadata: creationTimestamp: "2026-07-10T19:21:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:35Z" name: raw-sklearn-e7135-predictor-69d867d788.18c103f90f857d66 namespace: kserve-ci-e2e-test resourceVersion: "20862" uid: 51821750-2b33-49ce-9233-af8b10cf07e3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-e7135-predictor namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: 7519c589-13fe-45ef-a54d-dea0fb12e866 kind: Event lastTimestamp: "2026-07-10T19:21:35Z" message: Scaled up replica set raw-sklearn-e7135-predictor-69d867d788 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:21:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:35Z" name: raw-sklearn-e7135-predictor.18c103f90c3b6394 namespace: kserve-ci-e2e-test resourceVersion: "20849" uid: c09f1615-fbc1-4b79-bcac-634367e18dc4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:21:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-e7135-predictor namespace: kserve-ci-e2e-test resourceVersion: "20847" uid: 685c58bc-4a5f-424a-acf6-ac8c5e191510 kind: Event lastTimestamp: "2026-07-10T19:22:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:21:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:22:20Z" name: raw-sklearn-e7135-predictor.18c103fc8b25a479 namespace: kserve-ci-e2e-test resourceVersion: "21207" uid: 8441b891-498f-4d73-925d-43c5a8b322f5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:21:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-e7135-predictor namespace: kserve-ci-e2e-test resourceVersion: "20847" uid: 685c58bc-4a5f-424a-acf6-ac8c5e191510 kind: Event lastTimestamp: "2026-07-10T19:22:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:21:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:22:20Z" name: raw-sklearn-e7135-predictor.18c103fc8b7c15e8 namespace: kserve-ci-e2e-test resourceVersion: "21208" uid: 491df990-2074-4622-b043-0fc158467be9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:22:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-e7135-predictor namespace: kserve-ci-e2e-test resourceVersion: "20995" uid: 685c58bc-4a5f-424a-acf6-ac8c5e191510 kind: Event lastTimestamp: "2026-07-10T19:23:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:22:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:23:05Z" name: raw-sklearn-e7135-predictor.18c104070855d132 namespace: kserve-ci-e2e-test resourceVersion: "21529" uid: a833b55e-a50d-451b-8e2f-6f01990d85f4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:22:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-e7135-predictor namespace: kserve-ci-e2e-test resourceVersion: "20995" uid: 685c58bc-4a5f-424a-acf6-ac8c5e191510 kind: Event lastTimestamp: "2026-07-10T19:23:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:22:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:23:05Z" name: raw-sklearn-e7135-predictor.18c1040708bae799 namespace: kserve-ci-e2e-test resourceVersion: "21530" uid: e4413b24-09a3-44a5-aa6f-bdc07a091a2a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-10T19:21:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-e7135 namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: 9f57fe5c-81f0-4e72-9aef-4d8e47fbf10c kind: Event lastTimestamp: "2026-07-10T19:22:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:21:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:22:58Z" name: raw-sklearn-e7135.18c103f90c3ef0ee namespace: kserve-ci-e2e-test resourceVersion: "21476" uid: 0b494ccb-b442-4e30-89f4-0403616e3a23 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:21:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-e7135 namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: 9f57fe5c-81f0-4e72-9aef-4d8e47fbf10c kind: Event lastTimestamp: "2026-07-10T19:21:35Z" message: 'Failed to update status for InferenceService "raw-sklearn-e7135": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-e7135": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:21:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:21:35Z" name: raw-sklearn-e7135.18c103f90c8f2b42 namespace: kserve-ci-e2e-test resourceVersion: "20854" uid: f44b133e-7aa7-49fd-b537-e96f61e7301c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-e7135 namespace: kserve-ci-e2e-test resourceVersion: "21474" uid: 9f57fe5c-81f0-4e72-9aef-4d8e47fbf10c kind: Event lastTimestamp: "2026-07-10T19:22:58Z" message: InferenceService [raw-sklearn-e7135] is Ready metadata: creationTimestamp: "2026-07-10T19:22:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:22:58Z" name: raw-sklearn-e7135.18c1040c4170fa2e namespace: kserve-ci-e2e-test resourceVersion: "21475" uid: 0c80770a-d54c-48ac-a87d-5eddf2cc211f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:23:10.983454Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws namespace: kserve-ci-e2e-test resourceVersion: "21611" uid: 673e67ee-c25c-47c3-96af-ea55ce890671 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws to ip-10-0-134-26.ec2.internal metadata: creationTimestamp: "2026-07-10T19:23:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:23:10Z" name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws.18c1040f41632089 namespace: kserve-ci-e2e-test resourceVersion: "21615" uid: 9501abe6-a876-4057-84f2-2589011b9234 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c9dd4b6f4-9plfp source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:23:11Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: 673e67ee-c25c-47c3-96af-ea55ce890671 kind: Event lastTimestamp: "2026-07-10T19:23:11Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:23:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:23:11Z" name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws.18c1040f5b981bde namespace: kserve-ci-e2e-test resourceVersion: "21633" uid: 8a897968-5be6-444b-994f-27edf09e8cea reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:23:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws namespace: kserve-ci-e2e-test resourceVersion: "21613" uid: 673e67ee-c25c-47c3-96af-ea55ce890671 kind: Event lastTimestamp: "2026-07-10T19:23:11Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7893a590791cc9f33e3a3207dcc1d0843778bbf1aee8b6f22a63e3aa3b91a2e1" already present on machine metadata: creationTimestamp: "2026-07-10T19:23:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:23:11Z" name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws.18c1040f5de28c66 namespace: kserve-ci-e2e-test resourceVersion: "21635" uid: f00fba61-8819-4474-acba-05997f212068 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:23:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws namespace: kserve-ci-e2e-test resourceVersion: "21613" uid: 673e67ee-c25c-47c3-96af-ea55ce890671 kind: Event lastTimestamp: "2026-07-10T19:23:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:23:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:23:11Z" name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws.18c1040f611276e1 namespace: kserve-ci-e2e-test resourceVersion: "21637" uid: a1a5ac09-d570-4cc6-a0d1-167a3d010ebb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:23:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws namespace: kserve-ci-e2e-test resourceVersion: "21613" uid: 673e67ee-c25c-47c3-96af-ea55ce890671 kind: Event lastTimestamp: "2026-07-10T19:23:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:23:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:23:11Z" name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws.18c1040f617f26e9 namespace: kserve-ci-e2e-test resourceVersion: "21638" uid: 5a8d6c93-576d-446c-ab02-aa852aa6000a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:23:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws namespace: kserve-ci-e2e-test resourceVersion: "21613" uid: 673e67ee-c25c-47c3-96af-ea55ce890671 kind: Event lastTimestamp: "2026-07-10T19:23:15Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:23:15Z" name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws.18c1041046cf0ce0 namespace: kserve-ci-e2e-test resourceVersion: "21669" uid: a9798e32-e130-4753-b071-bc13f7aa7053 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:23:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws namespace: kserve-ci-e2e-test resourceVersion: "21613" uid: 673e67ee-c25c-47c3-96af-ea55ce890671 kind: Event lastTimestamp: "2026-07-10T19:23:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:23:15Z" name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws.18c1041049ee1858 namespace: kserve-ci-e2e-test resourceVersion: "21673" uid: c3c3140f-5a57-4fb7-9e14-8a2afec6fd8b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:23:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws namespace: kserve-ci-e2e-test resourceVersion: "21613" uid: 673e67ee-c25c-47c3-96af-ea55ce890671 kind: Event lastTimestamp: "2026-07-10T19:23:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:23:15Z" name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws.18c104104a794dd3 namespace: kserve-ci-e2e-test resourceVersion: "21675" uid: fc281309-8f30-49b0-8c34-ebb72b5ef713 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:23:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws namespace: kserve-ci-e2e-test resourceVersion: "21613" uid: 673e67ee-c25c-47c3-96af-ea55ce890671 kind: Event lastTimestamp: "2026-07-10T19:23:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:23:15Z" name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws.18c104104a8276fb namespace: kserve-ci-e2e-test resourceVersion: "21676" uid: da5d526e-e4f0-46e9-8342-c98c9edcea08 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:23:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws namespace: kserve-ci-e2e-test resourceVersion: "21613" uid: 673e67ee-c25c-47c3-96af-ea55ce890671 kind: Event lastTimestamp: "2026-07-10T19:23:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:23:15Z" name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws.18c104104d2cc2e5 namespace: kserve-ci-e2e-test resourceVersion: "21678" uid: 9c7a4923-f9e7-435e-b908-67c4a03cbc3e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:23:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws namespace: kserve-ci-e2e-test resourceVersion: "21613" uid: 673e67ee-c25c-47c3-96af-ea55ce890671 kind: Event lastTimestamp: "2026-07-10T19:23:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:23:15Z" name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws.18c104104d9846c2 namespace: kserve-ci-e2e-test resourceVersion: "21679" uid: 3bcca221-da2e-4111-9130-d9847fa210aa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-10T19:23:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws namespace: kserve-ci-e2e-test resourceVersion: "21613" uid: 673e67ee-c25c-47c3-96af-ea55ce890671 kind: Event lastTimestamp: "2026-07-10T19:24:43Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:23:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:24:43Z" name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws.18c10410be974d72 namespace: kserve-ci-e2e-test resourceVersion: "22329" uid: 18110d48-0d2e-41c2-9ce7-d6d12f9fa86a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:24:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws namespace: kserve-ci-e2e-test resourceVersion: "21613" uid: 673e67ee-c25c-47c3-96af-ea55ce890671 kind: Event lastTimestamp: "2026-07-10T19:24:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:24:41Z" name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws.18c104243d0cb8d0 namespace: kserve-ci-e2e-test resourceVersion: "22309" uid: a4c0d9b3-ecd9-47f2-a512-07b32d614803 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:24:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws namespace: kserve-ci-e2e-test resourceVersion: "21613" uid: 673e67ee-c25c-47c3-96af-ea55ce890671 kind: Event lastTimestamp: "2026-07-10T19:24:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:24:41Z" name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws.18c104243d0d7471 namespace: kserve-ci-e2e-test resourceVersion: "22310" uid: 70277d21-854c-46d3-ba6a-0dd2d73e2b87 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:24:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws namespace: kserve-ci-e2e-test resourceVersion: "21613" uid: 673e67ee-c25c-47c3-96af-ea55ce890671 kind: Event lastTimestamp: "2026-07-10T19:24:43Z" message: 'Readiness probe failed: Get "https://10.133.0.37:8643/healthz": dial tcp 10.133.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:24:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:24:43Z" name: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws.18c10424c4ca0eec namespace: kserve-ci-e2e-test resourceVersion: "22328" uid: cabdc478-9dbe-44a1-beae-5b38472c0124 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-26.ec2.internal source: component: kubelet host: ip-10-0-134-26.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:23:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-759ec-predictor-547bc74f8d namespace: kserve-ci-e2e-test resourceVersion: "21600" uid: 975f445f-91b8-4126-a342-85cd0aa64313 kind: Event lastTimestamp: "2026-07-10T19:23:10Z" message: 'Created pod: raw-sklearn-runtime-759ec-predictor-547bc74f8d-lsdws' metadata: creationTimestamp: "2026-07-10T19:23:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:23:10Z" name: raw-sklearn-runtime-759ec-predictor-547bc74f8d.18c1040f40f6cec3 namespace: kserve-ci-e2e-test resourceVersion: "21612" uid: 402e299a-1e60-498a-a994-b9a8d42011f1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:23:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-759ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "21597" uid: 9788f80b-8e15-4b43-945a-8e54dd29f351 kind: Event lastTimestamp: "2026-07-10T19:23:10Z" message: Scaled up replica set raw-sklearn-runtime-759ec-predictor-547bc74f8d from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:23:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:23:10Z" name: raw-sklearn-runtime-759ec-predictor.18c1040f3c5bb6ca namespace: kserve-ci-e2e-test resourceVersion: "21602" uid: d34c3d0d-9319-4b25-a3b2-3063b493d9fd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:23:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-759ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "21599" uid: b3de2762-b5f6-421e-9cd4-716ff48b4e5c kind: Event lastTimestamp: "2026-07-10T19:23:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:23:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:23:40Z" name: raw-sklearn-runtime-759ec-predictor.18c10412bb5a27a8 namespace: kserve-ci-e2e-test resourceVersion: "21860" uid: ac034877-29d3-4aee-b167-2ca09cae070b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:23:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-759ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "21599" uid: b3de2762-b5f6-421e-9cd4-716ff48b4e5c kind: Event lastTimestamp: "2026-07-10T19:23:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:23:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:23:40Z" name: raw-sklearn-runtime-759ec-predictor.18c10412bbb69759 namespace: kserve-ci-e2e-test resourceVersion: "21861" uid: 61ca044b-82db-44aa-85db-98b42e8aa256 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:23:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-759ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "21760" uid: b3de2762-b5f6-421e-9cd4-716ff48b4e5c kind: Event lastTimestamp: "2026-07-10T19:24:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:23:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:24:41Z" name: raw-sklearn-runtime-759ec-predictor.18c10419b9961b9b namespace: kserve-ci-e2e-test resourceVersion: "22291" uid: 0d51c328-fc52-4cef-9453-f5f0b335ec45 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:23:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-759ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "21760" uid: b3de2762-b5f6-421e-9cd4-716ff48b4e5c kind: Event lastTimestamp: "2026-07-10T19:24:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:23:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:24:41Z" name: raw-sklearn-runtime-759ec-predictor.18c10419ba101bd4 namespace: kserve-ci-e2e-test resourceVersion: "22292" uid: e94bb061-2064-4d32-885f-e059338c474b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-10T19:23:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-759ec namespace: kserve-ci-e2e-test resourceVersion: "21590" uid: 844d1447-a772-45ba-a59c-cfe90b798e4d kind: Event lastTimestamp: "2026-07-10T19:24:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:23:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:24:33Z" name: raw-sklearn-runtime-759ec.18c1040f3c5d431b namespace: kserve-ci-e2e-test resourceVersion: "22237" uid: 4ea8620a-c37c-4896-9d86-c389c9c12a44 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:23:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-759ec namespace: kserve-ci-e2e-test resourceVersion: "21590" uid: 844d1447-a772-45ba-a59c-cfe90b798e4d kind: Event lastTimestamp: "2026-07-10T19:23:11Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-759ec": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-759ec": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:23:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:23:11Z" name: raw-sklearn-runtime-759ec.18c1040f3d1d577f namespace: kserve-ci-e2e-test resourceVersion: "21624" uid: 9e5680f7-d1e7-4c0c-bd53-cd1dfdac46ed reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:23:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-759ec namespace: kserve-ci-e2e-test resourceVersion: "21594" uid: 844d1447-a772-45ba-a59c-cfe90b798e4d kind: Event lastTimestamp: "2026-07-10T19:23:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-759ec": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:23:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:23:11Z" name: raw-sklearn-runtime-759ec.18c1040f433d8612 namespace: kserve-ci-e2e-test resourceVersion: "21625" uid: c0f711e0-fa4c-42fa-a96a-7a6eb125b6e4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:24:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-759ec namespace: kserve-ci-e2e-test resourceVersion: "22234" uid: 844d1447-a772-45ba-a59c-cfe90b798e4d kind: Event lastTimestamp: "2026-07-10T19:24:33Z" message: InferenceService [raw-sklearn-runtime-759ec] is Ready metadata: creationTimestamp: "2026-07-10T19:24:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:24:33Z" name: raw-sklearn-runtime-759ec.18c10422773f064a namespace: kserve-ci-e2e-test resourceVersion: "22235" uid: d40f8108-3748-4b9f-b2f5-fc90fc86af66 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "22552"