--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:07:38.991461Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-2dd56b-predictor-7fd4584776-k9xds namespace: kserve-ci-e2e-test resourceVersion: "20489" uid: f3bf6eed-7619-4c20-b1c1-78d48e3c6e5b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-2dd56b-predictor-7fd4584776-k9xds to ip-10-0-135-188.ec2.internal metadata: creationTimestamp: "2026-07-01T21:07:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:07:38Z" name: isvc-init-fail-2dd56b-predictor-7fd4584776-k9xds.18be46898800214b namespace: kserve-ci-e2e-test resourceVersion: "20495" uid: 6bbb8d5f-91b2-4d7f-946d-cbe14403ba6c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dc7469c8-ljvgg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:39Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-2dd56b-predictor-7fd4584776-k9xds namespace: kserve-ci-e2e-test resourceVersion: "20499" uid: f3bf6eed-7619-4c20-b1c1-78d48e3c6e5b kind: Event lastTimestamp: "2026-07-01T21:07:39Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:07:39Z" name: isvc-init-fail-2dd56b-predictor-7fd4584776-k9xds.18be4689a19ba5c2 namespace: kserve-ci-e2e-test resourceVersion: "20507" uid: 6fc5317a-f404-4998-bca3-c34bab858823 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:07:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-2dd56b-predictor-7fd4584776-k9xds namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: f3bf6eed-7619-4c20-b1c1-78d48e3c6e5b kind: Event lastTimestamp: "2026-07-01T21:07:44Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cc7b8fac71b36ebfd1661675b86a84ade93db501973e87758605bd27f47937cb" already present on machine metadata: creationTimestamp: "2026-07-01T21:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:44Z" name: isvc-init-fail-2dd56b-predictor-7fd4584776-k9xds.18be4689a320d5ee namespace: kserve-ci-e2e-test resourceVersion: "20552" uid: 4673d737-9700-4929-ab8b-94cf0bc856b0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:07:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-2dd56b-predictor-7fd4584776-k9xds namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: f3bf6eed-7619-4c20-b1c1-78d48e3c6e5b kind: Event lastTimestamp: "2026-07-01T21:07:44Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T21:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:44Z" name: isvc-init-fail-2dd56b-predictor-7fd4584776-k9xds.18be4689a603a1d2 namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: 46df8e8a-c529-45b3-9343-37afe9459573 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:07:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-2dd56b-predictor-7fd4584776-k9xds namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: f3bf6eed-7619-4c20-b1c1-78d48e3c6e5b kind: Event lastTimestamp: "2026-07-01T21:07:44Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T21:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:44Z" name: isvc-init-fail-2dd56b-predictor-7fd4584776-k9xds.18be4689a6754430 namespace: kserve-ci-e2e-test resourceVersion: "20557" uid: 59fe723e-0ea3-4da9-bdca-a8f875630769 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-2dd56b-predictor-7fd4584776-k9xds namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: f3bf6eed-7619-4c20-b1c1-78d48e3c6e5b kind: Event lastTimestamp: "2026-07-01T21:07:48Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-07-01T21:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:48Z" name: isvc-init-fail-2dd56b-predictor-7fd4584776-k9xds.18be468bda2e7130 namespace: kserve-ci-e2e-test resourceVersion: "20607" uid: 20314a6e-2250-4b7c-86d5-4de5cb5b0af7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-2dd56b-predictor-7fd4584776 namespace: kserve-ci-e2e-test resourceVersion: "20482" uid: ebb5c95c-30ce-4dc8-80e9-1a30953a0ea0 kind: Event lastTimestamp: "2026-07-01T21:07:38Z" message: 'Created pod: isvc-init-fail-2dd56b-predictor-7fd4584776-k9xds' metadata: creationTimestamp: "2026-07-01T21:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:38Z" name: isvc-init-fail-2dd56b-predictor-7fd4584776.18be4689876de168 namespace: kserve-ci-e2e-test resourceVersion: "20493" uid: 3772ef81-cf49-4069-8b61-e6963fa51ac4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-2dd56b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20480" uid: 1ae1de85-c5c9-4ec7-a4e6-2e9a2e09c685 kind: Event lastTimestamp: "2026-07-01T21:07:38Z" message: Scaled up replica set isvc-init-fail-2dd56b-predictor-7fd4584776 from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:38Z" name: isvc-init-fail-2dd56b-predictor.18be4689846c6709 namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: e3534f35-3ae9-46a8-bf45-dfbfd6055478 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T21:07:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-2dd56b namespace: kserve-ci-e2e-test resourceVersion: "20473" uid: 2513139e-b381-48ad-a2cb-61334485b2ab kind: Event lastTimestamp: "2026-07-01T21:07:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T21:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:07:44Z" name: isvc-init-fail-2dd56b.18be46898472f609 namespace: kserve-ci-e2e-test resourceVersion: "20558" uid: acf83075-e3d3-4ef0-810f-c27f4a0a9c5b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-2dd56b namespace: kserve-ci-e2e-test resourceVersion: "20473" uid: 2513139e-b381-48ad-a2cb-61334485b2ab kind: Event lastTimestamp: "2026-07-01T21:07:38Z" message: 'Failed to update status for InferenceService "isvc-init-fail-2dd56b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-2dd56b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:07:38Z" name: isvc-init-fail-2dd56b.18be468984f0b56b namespace: kserve-ci-e2e-test resourceVersion: "20486" uid: 342b3503-29b8-45e9-b732-b32286c6fe73 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-2dd56b namespace: kserve-ci-e2e-test resourceVersion: "20473" uid: 2513139e-b381-48ad-a2cb-61334485b2ab kind: Event lastTimestamp: "2026-07-01T21:07:38Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-2dd56b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:07:38Z" name: isvc-init-fail-2dd56b.18be468984f126ef namespace: kserve-ci-e2e-test resourceVersion: "20487" uid: 2e427bd1-18a3-4cab-9721-4ad6817d962f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:01:45.031268Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997 namespace: kserve-ci-e2e-test resourceVersion: "17582" uid: 52b4c1eb-942a-4812-8d7d-96dd48e6635e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-1255c-predictor-65576c9ff7-gl997 to ip-10-0-135-188.ec2.internal metadata: creationTimestamp: "2026-07-01T21:01:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:01:45Z" name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997.18be46371e544244 namespace: kserve-ci-e2e-test resourceVersion: "17586" uid: 69ae8033-1f66-47a2-aceb-2f8b9e6e1c8e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dc7469c8-ljvgg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:45Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997 namespace: kserve-ci-e2e-test resourceVersion: "17591" uid: 52b4c1eb-942a-4812-8d7d-96dd48e6635e kind: Event lastTimestamp: "2026-07-01T21:01:45Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:01:45Z" name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997.18be46373798904a namespace: kserve-ci-e2e-test resourceVersion: "17601" uid: ab8ffe3e-b9cd-4dc9-8abc-7abdf7389d16 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 52b4c1eb-942a-4812-8d7d-96dd48e6635e kind: Event lastTimestamp: "2026-07-01T21:01:45Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cc7b8fac71b36ebfd1661675b86a84ade93db501973e87758605bd27f47937cb" already present on machine metadata: creationTimestamp: "2026-07-01T21:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:45Z" name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997.18be46373912349d namespace: kserve-ci-e2e-test resourceVersion: "17603" uid: e59b6d61-1ddd-4976-8dac-e0bc7de7a47a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 52b4c1eb-942a-4812-8d7d-96dd48e6635e kind: Event lastTimestamp: "2026-07-01T21:01:45Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T21:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:45Z" name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997.18be46373b7dba8f namespace: kserve-ci-e2e-test resourceVersion: "17604" uid: 6de74c50-911c-47ce-a09f-cd1541765f30 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 52b4c1eb-942a-4812-8d7d-96dd48e6635e kind: Event lastTimestamp: "2026-07-01T21:01:45Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T21:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:45Z" name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997.18be46373be692a4 namespace: kserve-ci-e2e-test resourceVersion: "17605" uid: f9ccbf44-46c9-4e9d-a0bd-599e83b4e8bc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 52b4c1eb-942a-4812-8d7d-96dd48e6635e kind: Event lastTimestamp: "2026-07-01T21:01:49Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-01T21:01:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:49Z" name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997.18be463817b48fc8 namespace: kserve-ci-e2e-test resourceVersion: "17642" uid: a67c5c42-aa2b-460d-96f0-4842c352f669 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 52b4c1eb-942a-4812-8d7d-96dd48e6635e kind: Event lastTimestamp: "2026-07-01T21:01:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T21:01:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:49Z" name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997.18be46381a595a98 namespace: kserve-ci-e2e-test resourceVersion: "17645" uid: 05e2dcf1-9623-4d01-838a-1791262958fa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 52b4c1eb-942a-4812-8d7d-96dd48e6635e kind: Event lastTimestamp: "2026-07-01T21:01:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T21:01:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:49Z" name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997.18be46381ac654b3 namespace: kserve-ci-e2e-test resourceVersion: "17646" uid: 421f16e1-cb9b-4ec3-b309-631248e2a7d6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 52b4c1eb-942a-4812-8d7d-96dd48e6635e kind: Event lastTimestamp: "2026-07-01T21:01:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T21:01:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:49Z" name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997.18be46381ad15ccd namespace: kserve-ci-e2e-test resourceVersion: "17647" uid: fd7ef749-6ce5-4f48-972a-f291b46eaa8a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 52b4c1eb-942a-4812-8d7d-96dd48e6635e kind: Event lastTimestamp: "2026-07-01T21:01:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T21:01:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:49Z" name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997.18be46381e365077 namespace: kserve-ci-e2e-test resourceVersion: "17651" uid: 53c0477c-6b81-45ca-a004-24bef616347e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 52b4c1eb-942a-4812-8d7d-96dd48e6635e kind: Event lastTimestamp: "2026-07-01T21:01:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:01:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:49Z" name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997.18be46381ec5ff29 namespace: kserve-ci-e2e-test resourceVersion: "17652" uid: db35fdfe-1239-4a80-b098-1c2dbc17aefe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 52b4c1eb-942a-4812-8d7d-96dd48e6635e kind: Event lastTimestamp: "2026-07-01T21:01:49Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:eb91db6613fb4bde95991e8e02d444548e2e6e23bd5e4e0a11c1f350f3203183" already present on machine metadata: creationTimestamp: "2026-07-01T21:01:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:49Z" name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997.18be46381ed1196a namespace: kserve-ci-e2e-test resourceVersion: "17653" uid: 1e77863f-1050-4526-a35d-7557b06793b5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 52b4c1eb-942a-4812-8d7d-96dd48e6635e kind: Event lastTimestamp: "2026-07-01T21:01:49Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-01T21:01:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:49Z" name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997.18be463821822dd6 namespace: kserve-ci-e2e-test resourceVersion: "17654" uid: 6ff27c80-61ab-47fd-a153-77e76c4bee46 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 52b4c1eb-942a-4812-8d7d-96dd48e6635e kind: Event lastTimestamp: "2026-07-01T21:01:49Z" message: Started container agent metadata: creationTimestamp: "2026-07-01T21:01:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:49Z" name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997.18be463821ecd017 namespace: kserve-ci-e2e-test resourceVersion: "17655" uid: 526f1469-868e-456e-899a-91c928add549 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-01T21:01:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 52b4c1eb-942a-4812-8d7d-96dd48e6635e kind: Event lastTimestamp: "2026-07-01T21:03:36Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:01:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:03:36Z" name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997.18be463853b1a37e namespace: kserve-ci-e2e-test resourceVersion: "18552" uid: 5213ef83-77f4-4612-b062-ee789eb8339c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T21:01:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 52b4c1eb-942a-4812-8d7d-96dd48e6635e kind: Event lastTimestamp: "2026-07-01T21:03:26Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T21:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:03:26Z" name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997.18be46388f8a8572 namespace: kserve-ci-e2e-test resourceVersion: "18469" uid: f9ff7735-64d0-49f9-a4b1-bd9c960c7333 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:03:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 52b4c1eb-942a-4812-8d7d-96dd48e6635e kind: Event lastTimestamp: "2026-07-01T21:03:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T21:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:03:20Z" name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997.18be464d4fce5b4a namespace: kserve-ci-e2e-test resourceVersion: "18366" uid: 151df4a9-609a-4397-ae7c-0581fff1e70a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:03:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 52b4c1eb-942a-4812-8d7d-96dd48e6635e kind: Event lastTimestamp: "2026-07-01T21:03:20Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-01T21:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:03:20Z" name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997.18be464d4fcf158a namespace: kserve-ci-e2e-test resourceVersion: "18370" uid: 1d8cc8f9-7bc2-4cc2-b7e8-7e2f95c0ed82 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:03:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 52b4c1eb-942a-4812-8d7d-96dd48e6635e kind: Event lastTimestamp: "2026-07-01T21:03:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:03:20Z" name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997.18be464d4fcf9ba3 namespace: kserve-ci-e2e-test resourceVersion: "18377" uid: 8cf7ec11-23b0-4ec6-825a-fbd7f21b9fa0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:03:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: 52b4c1eb-942a-4812-8d7d-96dd48e6635e kind: Event lastTimestamp: "2026-07-01T21:03:32Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:03:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:03:32Z" name: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997.18be464dbfc7caf2 namespace: kserve-ci-e2e-test resourceVersion: "18515" uid: c61f0ac2-aee5-4d1d-9cf7-d74656f523d6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-1255c-predictor-65576c9ff7 namespace: kserve-ci-e2e-test resourceVersion: "17568" uid: 2a773a16-f6c8-4126-80cf-23ecb0012a41 kind: Event lastTimestamp: "2026-07-01T21:01:45Z" message: 'Created pod: isvc-logger-raw-1255c-predictor-65576c9ff7-gl997' metadata: creationTimestamp: "2026-07-01T21:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:01:45Z" name: isvc-logger-raw-1255c-predictor-65576c9ff7.18be46371de17f06 namespace: kserve-ci-e2e-test resourceVersion: "17583" uid: 955b78ed-6c8c-4081-b71c-8709e56b4852 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-1255c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17566" uid: 9b18335c-370b-4cba-98fe-b6c269ce80a6 kind: Event lastTimestamp: "2026-07-01T21:01:44Z" message: Scaled up replica set isvc-logger-raw-1255c-predictor-65576c9ff7 from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:01:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:01:44Z" name: isvc-logger-raw-1255c-predictor.18be46371a3b3708 namespace: kserve-ci-e2e-test resourceVersion: "17569" uid: e011fe20-ebd3-4db8-bbb5-65c4f394e7ff reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:01:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-1255c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17567" uid: 7362ad73-98a7-496d-b6ca-48ca5044a78a kind: Event lastTimestamp: "2026-07-01T21:02:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:01:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:02:30Z" name: isvc-logger-raw-1255c-predictor.18be463a98a6b34c namespace: kserve-ci-e2e-test resourceVersion: "17958" uid: c2e080b2-c5ae-4327-8437-6a123664ff36 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:01:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-1255c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17567" uid: 7362ad73-98a7-496d-b6ca-48ca5044a78a kind: Event lastTimestamp: "2026-07-01T21:02:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:01:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:02:30Z" name: isvc-logger-raw-1255c-predictor.18be463a9908f562 namespace: kserve-ci-e2e-test resourceVersion: "17959" uid: 67df5e70-903e-4778-bc1f-6d98458130d8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:02:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-1255c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17740" uid: 7362ad73-98a7-496d-b6ca-48ca5044a78a kind: Event lastTimestamp: "2026-07-01T21:03:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:03:15Z" name: isvc-logger-raw-1255c-predictor.18be46451551dab7 namespace: kserve-ci-e2e-test resourceVersion: "18285" uid: a87e13bc-7af1-4f7e-8513-4b7ac91dcdfd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:02:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-1255c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17740" uid: 7362ad73-98a7-496d-b6ca-48ca5044a78a kind: Event lastTimestamp: "2026-07-01T21:03:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:03:15Z" name: isvc-logger-raw-1255c-predictor.18be464515b4c445 namespace: kserve-ci-e2e-test resourceVersion: "18286" uid: 4492a2f7-b2bf-4ebe-991a-d88628269984 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T21:01:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-1255c namespace: kserve-ci-e2e-test resourceVersion: "17559" uid: 8104313f-9cbf-4664-a9d0-2b506e382354 kind: Event lastTimestamp: "2026-07-01T21:03:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T21:01:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:03:06Z" name: isvc-logger-raw-1255c.18be46371a386e91 namespace: kserve-ci-e2e-test resourceVersion: "18228" uid: d322e42f-4174-451e-9e0a-8c9bfa7824b3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:01:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-1255c namespace: kserve-ci-e2e-test resourceVersion: "17559" uid: 8104313f-9cbf-4664-a9d0-2b506e382354 kind: Event lastTimestamp: "2026-07-01T21:01:45Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-1255c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-1255c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:01:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:01:45Z" name: isvc-logger-raw-1255c.18be46371a89a48f namespace: kserve-ci-e2e-test resourceVersion: "17593" uid: 83924001-a3b8-44d9-b30f-5bd3e8abc022 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-1255c namespace: kserve-ci-e2e-test resourceVersion: "17561" uid: 8104313f-9cbf-4664-a9d0-2b506e382354 kind: Event lastTimestamp: "2026-07-01T21:01:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-1255c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:01:45Z" name: isvc-logger-raw-1255c.18be4637211bb62e namespace: kserve-ci-e2e-test resourceVersion: "17594" uid: e92f642c-c52b-4ed2-ac36-ac431431fcbe reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:03:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-1255c namespace: kserve-ci-e2e-test resourceVersion: "18225" uid: 8104313f-9cbf-4664-a9d0-2b506e382354 kind: Event lastTimestamp: "2026-07-01T21:03:06Z" message: InferenceService [isvc-logger-raw-1255c] is Ready metadata: creationTimestamp: "2026-07-01T21:03:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:03:06Z" name: isvc-logger-raw-1255c.18be464a1fbfad39 namespace: kserve-ci-e2e-test resourceVersion: "18226" uid: 5559403f-edb6-40b7-aae8-92a6357be8a8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:05:55.634687Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f namespace: kserve-ci-e2e-test resourceVersion: "19576" uid: 660ad75b-aeb2-4663-ab29-a21312b52718 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-77d927-predictor-86f44cb6d7-qvq7f to ip-10-0-135-188.ec2.internal metadata: creationTimestamp: "2026-07-01T21:05:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:05:55Z" name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f.18be46717774f269 namespace: kserve-ci-e2e-test resourceVersion: "19581" uid: ff901156-bb0f-4d7b-add3-86f337b1eead reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dc7469c8-ljvgg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:56Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f namespace: kserve-ci-e2e-test resourceVersion: "19583" uid: 660ad75b-aeb2-4663-ab29-a21312b52718 kind: Event lastTimestamp: "2026-07-01T21:05:56Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:05:56Z" name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f.18be467190820517 namespace: kserve-ci-e2e-test resourceVersion: "19593" uid: d0e592ac-eed2-4c3e-ba4f-cf245ccd3fe6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f namespace: kserve-ci-e2e-test resourceVersion: "19579" uid: 660ad75b-aeb2-4663-ab29-a21312b52718 kind: Event lastTimestamp: "2026-07-01T21:05:56Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cc7b8fac71b36ebfd1661675b86a84ade93db501973e87758605bd27f47937cb" already present on machine metadata: creationTimestamp: "2026-07-01T21:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:56Z" name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f.18be46719318766d namespace: kserve-ci-e2e-test resourceVersion: "19596" uid: 053bb5a8-bb22-4dce-b820-621e71920934 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f namespace: kserve-ci-e2e-test resourceVersion: "19579" uid: 660ad75b-aeb2-4663-ab29-a21312b52718 kind: Event lastTimestamp: "2026-07-01T21:05:56Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T21:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:56Z" name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f.18be467196726877 namespace: kserve-ci-e2e-test resourceVersion: "19597" uid: fb128441-6031-4639-87bd-27e02be6ab2d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f namespace: kserve-ci-e2e-test resourceVersion: "19579" uid: 660ad75b-aeb2-4663-ab29-a21312b52718 kind: Event lastTimestamp: "2026-07-01T21:05:56Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T21:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:56Z" name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f.18be467196e5bf97 namespace: kserve-ci-e2e-test resourceVersion: "19598" uid: ca5c2a6e-8d4b-4aaf-9976-655dc65fb43d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f namespace: kserve-ci-e2e-test resourceVersion: "19579" uid: 660ad75b-aeb2-4663-ab29-a21312b52718 kind: Event lastTimestamp: "2026-07-01T21:06:00Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-01T21:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:00Z" name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f.18be467287555967 namespace: kserve-ci-e2e-test resourceVersion: "19635" uid: 3ca064b4-9732-4c02-abd2-c338716945a1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f namespace: kserve-ci-e2e-test resourceVersion: "19579" uid: 660ad75b-aeb2-4663-ab29-a21312b52718 kind: Event lastTimestamp: "2026-07-01T21:06:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T21:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:00Z" name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f.18be46728a98122a namespace: kserve-ci-e2e-test resourceVersion: "19639" uid: 770d6718-f425-4a1a-ab95-b548e95fee3d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f namespace: kserve-ci-e2e-test resourceVersion: "19579" uid: 660ad75b-aeb2-4663-ab29-a21312b52718 kind: Event lastTimestamp: "2026-07-01T21:06:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T21:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:00Z" name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f.18be46728b0e5f7b namespace: kserve-ci-e2e-test resourceVersion: "19640" uid: 6abea2d2-b46f-4ddc-90e3-b2263b4a7cd5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f namespace: kserve-ci-e2e-test resourceVersion: "19579" uid: 660ad75b-aeb2-4663-ab29-a21312b52718 kind: Event lastTimestamp: "2026-07-01T21:06:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T21:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:00Z" name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f.18be46728b2262a8 namespace: kserve-ci-e2e-test resourceVersion: "19641" uid: 8093cecb-82d2-4da6-aa16-f257caf626b3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f namespace: kserve-ci-e2e-test resourceVersion: "19579" uid: 660ad75b-aeb2-4663-ab29-a21312b52718 kind: Event lastTimestamp: "2026-07-01T21:06:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T21:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:00Z" name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f.18be46728dad0049 namespace: kserve-ci-e2e-test resourceVersion: "19642" uid: 244ad7fd-4bba-4ab8-b8d6-927a9ecc75fc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f namespace: kserve-ci-e2e-test resourceVersion: "19579" uid: 660ad75b-aeb2-4663-ab29-a21312b52718 kind: Event lastTimestamp: "2026-07-01T21:06:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:06:00Z" name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f.18be46728e2e01a8 namespace: kserve-ci-e2e-test resourceVersion: "19643" uid: 7cf29c0b-c56a-410d-84c2-88eb2293a183 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T21:06:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f namespace: kserve-ci-e2e-test resourceVersion: "19579" uid: 660ad75b-aeb2-4663-ab29-a21312b52718 kind: Event lastTimestamp: "2026-07-01T21:07:07Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:06:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:07Z" name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f.18be4672c3acb7e4 namespace: kserve-ci-e2e-test resourceVersion: "20124" uid: 42bd15ee-258e-451b-ba89-7fa316030c03 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f namespace: kserve-ci-e2e-test resourceVersion: "19579" uid: 660ad75b-aeb2-4663-ab29-a21312b52718 kind: Event lastTimestamp: "2026-07-01T21:07:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T21:07:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:33Z" name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f.18be468854bf96e1 namespace: kserve-ci-e2e-test resourceVersion: "20419" uid: b4b4f904-744a-4bfd-b903-b63aea9b8d91 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f namespace: kserve-ci-e2e-test resourceVersion: "19579" uid: 660ad75b-aeb2-4663-ab29-a21312b52718 kind: Event lastTimestamp: "2026-07-01T21:07:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:07:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:33Z" name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f.18be468854c010e9 namespace: kserve-ci-e2e-test resourceVersion: "20420" uid: 3351528e-34ad-405e-beca-381b5d759a5e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f namespace: kserve-ci-e2e-test resourceVersion: "19579" uid: 660ad75b-aeb2-4663-ab29-a21312b52718 kind: Event lastTimestamp: "2026-07-01T21:07:37Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:37Z" name: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f.18be46891e082688 namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: 67d5a507-f842-4bae-94b1-f7fc1f0d3171 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-77d927-predictor-86f44cb6d7 namespace: kserve-ci-e2e-test resourceVersion: "19565" uid: a8b2bd52-d81c-4aa9-a1c9-08bb3d688626 kind: Event lastTimestamp: "2026-07-01T21:05:55Z" message: 'Created pod: isvc-primary-77d927-predictor-86f44cb6d7-qvq7f' metadata: creationTimestamp: "2026-07-01T21:05:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:05:55Z" name: isvc-primary-77d927-predictor-86f44cb6d7.18be467175da44c7 namespace: kserve-ci-e2e-test resourceVersion: "19577" uid: c31d6c75-55ae-4cc1-b5e2-748d35fb8570 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-77d927-predictor namespace: kserve-ci-e2e-test resourceVersion: "19563" uid: ce4a31c0-df18-4d0d-a5a4-b84990074900 kind: Event lastTimestamp: "2026-07-01T21:05:55Z" message: Scaled up replica set isvc-primary-77d927-predictor-86f44cb6d7 from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:05:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:05:55Z" name: isvc-primary-77d927-predictor.18be467170fd8949 namespace: kserve-ci-e2e-test resourceVersion: "19566" uid: 2bfbe2a4-a12e-4375-bcc2-653766cd8ec5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-77d927-predictor namespace: kserve-ci-e2e-test resourceVersion: "19564" uid: 25573949-ea5f-434f-b6a4-42f36fc95420 kind: Event lastTimestamp: "2026-07-01T21:06:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:06:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:06:10Z" name: isvc-primary-77d927-predictor.18be4674f003c65f namespace: kserve-ci-e2e-test resourceVersion: "19727" uid: c3ebf773-1b57-4169-b5f3-662e0aea398e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:06:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-77d927-predictor namespace: kserve-ci-e2e-test resourceVersion: "19564" uid: 25573949-ea5f-434f-b6a4-42f36fc95420 kind: Event lastTimestamp: "2026-07-01T21:06:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:06:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:06:10Z" name: isvc-primary-77d927-predictor.18be4674f0706781 namespace: kserve-ci-e2e-test resourceVersion: "19729" uid: 5a4b147e-d8b7-41a8-8e16-8422815a6418 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T21:06:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-77d927-predictor namespace: kserve-ci-e2e-test resourceVersion: "19728" uid: 25573949-ea5f-434f-b6a4-42f36fc95420 kind: Event lastTimestamp: "2026-07-01T21:07:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:06:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:25Z" name: isvc-primary-77d927-predictor.18be46786f2994a8 namespace: kserve-ci-e2e-test resourceVersion: "20319" uid: 9f336314-239c-4cdf-b6b2-fbe501aa1c71 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T21:06:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-77d927-predictor namespace: kserve-ci-e2e-test resourceVersion: "19728" uid: 25573949-ea5f-434f-b6a4-42f36fc95420 kind: Event lastTimestamp: "2026-07-01T21:07:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:06:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:25Z" name: isvc-primary-77d927-predictor.18be46786fbb14d8 namespace: kserve-ci-e2e-test resourceVersion: "20320" uid: 6366ff5d-b6a1-4888-8000-ca5e3309c68f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T21:05:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-77d927 namespace: kserve-ci-e2e-test resourceVersion: "19556" uid: 9363148a-32a6-42f4-b614-1cd91618f38f kind: Event lastTimestamp: "2026-07-01T21:07:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T21:05:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:07:11Z" name: isvc-primary-77d927.18be467170f52ef7 namespace: kserve-ci-e2e-test resourceVersion: "20163" uid: 49a7a5ec-d643-4806-b400-f0e1027fa2fa reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-77d927 namespace: kserve-ci-e2e-test resourceVersion: "19556" uid: 9363148a-32a6-42f4-b614-1cd91618f38f kind: Event lastTimestamp: "2026-07-01T21:05:55Z" message: 'Failed to update status for InferenceService "isvc-primary-77d927": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-77d927": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:05:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:05:55Z" name: isvc-primary-77d927.18be46717167ca1e namespace: kserve-ci-e2e-test resourceVersion: "19569" uid: faf6c520-887e-442d-97b9-7e4acd6b8cde reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-77d927 namespace: kserve-ci-e2e-test resourceVersion: "19556" uid: 9363148a-32a6-42f4-b614-1cd91618f38f kind: Event lastTimestamp: "2026-07-01T21:05:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-77d927": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:05:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:05:55Z" name: isvc-primary-77d927.18be467171681550 namespace: kserve-ci-e2e-test resourceVersion: "19570" uid: 947698c8-f6b9-4675-aa00-ca443f40eeca reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-77d927 namespace: kserve-ci-e2e-test resourceVersion: "20161" uid: 9363148a-32a6-42f4-b614-1cd91618f38f kind: Event lastTimestamp: "2026-07-01T21:07:11Z" message: InferenceService [isvc-primary-77d927] is Ready metadata: creationTimestamp: "2026-07-01T21:07:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:07:11Z" name: isvc-primary-77d927.18be4683299f54a4 namespace: kserve-ci-e2e-test resourceVersion: "20162" uid: 5d548dae-41b7-4441-a389-4a73e3681826 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T20:55:59.265094Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh namespace: kserve-ci-e2e-test resourceVersion: "14436" uid: ea36b617-d104-4fc4-919a-8c2eafa8edf6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh to ip-10-0-135-188.ec2.internal metadata: creationTimestamp: "2026-07-01T20:55:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T20:55:59Z" name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh.18be45e69d0f28d9 namespace: kserve-ci-e2e-test resourceVersion: "14441" uid: 93af2f8e-f2d6-48dd-a8b4-7bc9c4f0567d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dc7469c8-ljvgg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh namespace: kserve-ci-e2e-test resourceVersion: "14448" uid: ea36b617-d104-4fc4-919a-8c2eafa8edf6 kind: Event lastTimestamp: "2026-07-01T20:55:59Z" message: Add eth0 [10.134.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T20:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T20:55:59Z" name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh.18be45e6b634f957 namespace: kserve-ci-e2e-test resourceVersion: "14458" uid: 5f8c924c-7c64-4489-870d-6dc5889dc764 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: ea36b617-d104-4fc4-919a-8c2eafa8edf6 kind: Event lastTimestamp: "2026-07-01T20:55:59Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cc7b8fac71b36ebfd1661675b86a84ade93db501973e87758605bd27f47937cb" already present on machine metadata: creationTimestamp: "2026-07-01T20:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:59Z" name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh.18be45e6b7d6bdbb namespace: kserve-ci-e2e-test resourceVersion: "14460" uid: 72539f30-4be5-473c-bddd-81eaa02ea173 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: ea36b617-d104-4fc4-919a-8c2eafa8edf6 kind: Event lastTimestamp: "2026-07-01T20:55:59Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T20:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:59Z" name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh.18be45e6bab22fa2 namespace: kserve-ci-e2e-test resourceVersion: "14461" uid: 1ac08703-feba-4764-b417-05f0ba279299 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: ea36b617-d104-4fc4-919a-8c2eafa8edf6 kind: Event lastTimestamp: "2026-07-01T20:55:59Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T20:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:55:59Z" name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh.18be45e6bb15a988 namespace: kserve-ci-e2e-test resourceVersion: "14462" uid: 9c488e32-be67-4485-8d93-70319bd6ed10 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: ea36b617-d104-4fc4-919a-8c2eafa8edf6 kind: Event lastTimestamp: "2026-07-01T20:56:03Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1683" metadata: creationTimestamp: "2026-07-01T20:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:03Z" name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh.18be45e7a9758b87 namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: e1542361-3212-433b-b866-7a5dd6c4c127 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: ea36b617-d104-4fc4-919a-8c2eafa8edf6 kind: Event lastTimestamp: "2026-07-01T20:56:18Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1683" in 14.431s (14.431s including waiting). Image size: 1555648931 bytes.' metadata: creationTimestamp: "2026-07-01T20:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:18Z" name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh.18be45eb05a780a3 namespace: kserve-ci-e2e-test resourceVersion: "14603" uid: 0b6cfa71-ba4b-4aec-870b-b4b6ecacd083 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: ea36b617-d104-4fc4-919a-8c2eafa8edf6 kind: Event lastTimestamp: "2026-07-01T20:56:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T20:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:18Z" name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh.18be45eb08d4076d namespace: kserve-ci-e2e-test resourceVersion: "14604" uid: b464ec6a-9891-47ba-b6cc-077a0d943d94 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: ea36b617-d104-4fc4-919a-8c2eafa8edf6 kind: Event lastTimestamp: "2026-07-01T20:56:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T20:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:18Z" name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh.18be45eb0948c136 namespace: kserve-ci-e2e-test resourceVersion: "14605" uid: 57891d38-3729-492d-b7da-535c3c72bdab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: ea36b617-d104-4fc4-919a-8c2eafa8edf6 kind: Event lastTimestamp: "2026-07-01T20:56:18Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T20:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:18Z" name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh.18be45eb0954d3ed namespace: kserve-ci-e2e-test resourceVersion: "14606" uid: 12ab0314-b52f-4ca4-b53d-0bfb7d915de2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: ea36b617-d104-4fc4-919a-8c2eafa8edf6 kind: Event lastTimestamp: "2026-07-01T20:56:20Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.343s (2.343s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T20:56:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:20Z" name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh.18be45eb95017e94 namespace: kserve-ci-e2e-test resourceVersion: "14621" uid: 8c997915-6c15-4891-88b5-0d28314542af reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: ea36b617-d104-4fc4-919a-8c2eafa8edf6 kind: Event lastTimestamp: "2026-07-01T20:56:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T20:56:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:20Z" name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh.18be45eb984433a6 namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: 9ff1b2db-6804-42cd-afda-d05ee92cce2e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: ea36b617-d104-4fc4-919a-8c2eafa8edf6 kind: Event lastTimestamp: "2026-07-01T20:56:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:56:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:20Z" name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh.18be45eb98bd5c7c namespace: kserve-ci-e2e-test resourceVersion: "14624" uid: 8712e6f5-3d33-4d7e-a215-5fc0558224d3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: ea36b617-d104-4fc4-919a-8c2eafa8edf6 kind: Event lastTimestamp: "2026-07-01T20:56:20Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:eb91db6613fb4bde95991e8e02d444548e2e6e23bd5e4e0a11c1f350f3203183" metadata: creationTimestamp: "2026-07-01T20:56:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:20Z" name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh.18be45eb98ca5518 namespace: kserve-ci-e2e-test resourceVersion: "14625" uid: e6b109f9-bb33-4822-9089-10c2c990e9a3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: ea36b617-d104-4fc4-919a-8c2eafa8edf6 kind: Event lastTimestamp: "2026-07-01T20:56:22Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:eb91db6613fb4bde95991e8e02d444548e2e6e23bd5e4e0a11c1f350f3203183" in 2.245s (2.245s including waiting). Image size: 239994455 bytes.' metadata: creationTimestamp: "2026-07-01T20:56:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:22Z" name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh.18be45ec1ea67bd2 namespace: kserve-ci-e2e-test resourceVersion: "14642" uid: ff95bb8f-9a55-4e05-8194-8e731769a9bc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: ea36b617-d104-4fc4-919a-8c2eafa8edf6 kind: Event lastTimestamp: "2026-07-01T20:56:22Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-01T20:56:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:22Z" name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh.18be45ec2185d1d5 namespace: kserve-ci-e2e-test resourceVersion: "14643" uid: add4e2b8-4dd5-41d1-9433-d3820c97d9a4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:56:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: ea36b617-d104-4fc4-919a-8c2eafa8edf6 kind: Event lastTimestamp: "2026-07-01T20:56:22Z" message: Started container agent metadata: creationTimestamp: "2026-07-01T20:56:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:56:22Z" name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh.18be45ec21f5bc0a namespace: kserve-ci-e2e-test resourceVersion: "14644" uid: b63e9631-8845-45af-9f9e-380b066f4397 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T20:56:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: ea36b617-d104-4fc4-919a-8c2eafa8edf6 kind: Event lastTimestamp: "2026-07-01T20:57:59Z" message: 'Readiness probe failed: dial tcp 10.134.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T20:56:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:59Z" name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh.18be45ec577fba8c namespace: kserve-ci-e2e-test resourceVersion: "15500" uid: 9a4c7e54-76c0-4946-beb3-874bebd0a241 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T20:56:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: ea36b617-d104-4fc4-919a-8c2eafa8edf6 kind: Event lastTimestamp: "2026-07-01T20:57:59Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T20:56:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:59Z" name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh.18be45ec5788f0e8 namespace: kserve-ci-e2e-test resourceVersion: "15501" uid: 9b57f8d2-f863-4792-aa72-306ae47fd36f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: ea36b617-d104-4fc4-919a-8c2eafa8edf6 kind: Event lastTimestamp: "2026-07-01T20:57:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T20:57:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:44Z" name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh.18be45ff1d0f3cfd namespace: kserve-ci-e2e-test resourceVersion: "15295" uid: 31dbb593-c90d-4829-a0be-ea50e7a4cdb7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: ea36b617-d104-4fc4-919a-8c2eafa8edf6 kind: Event lastTimestamp: "2026-07-01T20:57:44Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-01T20:57:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:44Z" name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh.18be45ff1d0f8fb2 namespace: kserve-ci-e2e-test resourceVersion: "15297" uid: a2e96fdc-56a7-4329-98e6-af86204a3225 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: ea36b617-d104-4fc4-919a-8c2eafa8edf6 kind: Event lastTimestamp: "2026-07-01T20:57:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:57:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:44Z" name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh.18be45ff1d0fa181 namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: 4a7f99b6-e640-417a-9be4-d3a5f8f842dd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T20:57:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: ea36b617-d104-4fc4-919a-8c2eafa8edf6 kind: Event lastTimestamp: "2026-07-01T20:58:04Z" message: 'Readiness probe failed: Get "https://10.134.0.19:8643/healthz": dial tcp 10.134.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T20:57:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:58:04Z" name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh.18be45ff33a189b9 namespace: kserve-ci-e2e-test resourceVersion: "15534" uid: 3405258d-ae65-435e-827d-6f2fd8586ba6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8 namespace: kserve-ci-e2e-test resourceVersion: "14424" uid: 00daf3cb-db73-4c32-a739-7d255f5973e9 kind: Event lastTimestamp: "2026-07-01T20:55:59Z" message: 'Created pod: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8-6wvjh' metadata: creationTimestamp: "2026-07-01T20:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:55:59Z" name: isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8.18be45e69c9e003d namespace: kserve-ci-e2e-test resourceVersion: "14437" uid: b7fa78ca-c4da-45bf-9ec5-bb78d14a95a0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:55:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-06549-predictor namespace: kserve-ci-e2e-test resourceVersion: "14423" uid: 3deda52e-f603-4589-9c53-a4cb9ff30caa kind: Event lastTimestamp: "2026-07-01T20:55:59Z" message: Scaled up replica set isvc-raw-sklearn-batcher-06549-predictor-5778c86cc8 from 0 to 1 metadata: creationTimestamp: "2026-07-01T20:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:55:59Z" name: isvc-raw-sklearn-batcher-06549-predictor.18be45e6945b436c namespace: kserve-ci-e2e-test resourceVersion: "14425" uid: b65d137d-7146-4498-8038-b631e73b202a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T20:56:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-06549-predictor namespace: kserve-ci-e2e-test resourceVersion: "14431" uid: 539f5186-a2f8-4321-bcff-f47f78cc69ea kind: Event lastTimestamp: "2026-07-01T20:56:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:56:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:56:59Z" name: isvc-raw-sklearn-batcher-06549-predictor.18be45ea19973a82 namespace: kserve-ci-e2e-test resourceVersion: "14912" uid: 1722af9a-e2a9-4908-aeb1-1d878ed21145 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T20:56:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-06549-predictor namespace: kserve-ci-e2e-test resourceVersion: "14431" uid: 539f5186-a2f8-4321-bcff-f47f78cc69ea kind: Event lastTimestamp: "2026-07-01T20:56:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:56:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:56:59Z" name: isvc-raw-sklearn-batcher-06549-predictor.18be45ea19fbbcca namespace: kserve-ci-e2e-test resourceVersion: "14913" uid: 1394a326-215e-459f-9007-1186ddb6088d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T20:56:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-06549-predictor namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: 539f5186-a2f8-4321-bcff-f47f78cc69ea kind: Event lastTimestamp: "2026-07-01T20:57:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T20:56:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:57:44Z" name: isvc-raw-sklearn-batcher-06549-predictor.18be45f117f5762e namespace: kserve-ci-e2e-test resourceVersion: "15253" uid: 2716ea9b-125e-479f-a419-c689e75853b5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T20:56:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-06549-predictor namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: 539f5186-a2f8-4321-bcff-f47f78cc69ea kind: Event lastTimestamp: "2026-07-01T20:57:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T20:56:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:57:44Z" name: isvc-raw-sklearn-batcher-06549-predictor.18be45f1185e9155 namespace: kserve-ci-e2e-test resourceVersion: "15256" uid: 74207858-9f3b-4ea9-b166-63d3a213c780 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-01T20:55:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-06549 namespace: kserve-ci-e2e-test resourceVersion: "14414" uid: ecbd767a-0d2e-485c-80e5-02dcdeaf494b kind: Event lastTimestamp: "2026-07-01T20:57:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T20:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:57:29Z" name: isvc-raw-sklearn-batcher-06549.18be45e69a622d01 namespace: kserve-ci-e2e-test resourceVersion: "15141" uid: c450e176-35f2-4cec-a227-a0e249337d69 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T20:55:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-06549 namespace: kserve-ci-e2e-test resourceVersion: "14414" uid: ecbd767a-0d2e-485c-80e5-02dcdeaf494b kind: Event lastTimestamp: "2026-07-01T20:55:59Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-06549": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-06549": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T20:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:55:59Z" name: isvc-raw-sklearn-batcher-06549.18be45e69ac7d90d namespace: kserve-ci-e2e-test resourceVersion: "14450" uid: 94b6dbbb-e753-4abd-acd0-19a8bbad0a92 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T20:55:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-06549 namespace: kserve-ci-e2e-test resourceVersion: "14414" uid: ecbd767a-0d2e-485c-80e5-02dcdeaf494b kind: Event lastTimestamp: "2026-07-01T20:55:59Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-06549": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T20:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:55:59Z" name: isvc-raw-sklearn-batcher-06549.18be45e69ac82f51 namespace: kserve-ci-e2e-test resourceVersion: "14451" uid: 91cbaf60-5dbb-46db-be35-26b2e0c8c5bd reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-06549 namespace: kserve-ci-e2e-test resourceVersion: "15138" uid: ecbd767a-0d2e-485c-80e5-02dcdeaf494b kind: Event lastTimestamp: "2026-07-01T20:57:29Z" message: InferenceService [isvc-raw-sklearn-batcher-06549] is Ready metadata: creationTimestamp: "2026-07-01T20:57:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:57:29Z" name: isvc-raw-sklearn-batcher-06549.18be45fbbaeb6ab0 namespace: kserve-ci-e2e-test resourceVersion: "15139" uid: 6ffc3950-9dd3-430f-a8e1-2a4734cf39ce reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:07:15.704173Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-77d927-predictor-67bbb9f4c6-cf6xf namespace: kserve-ci-e2e-test resourceVersion: "20216" uid: f2881f6d-1c8b-4dcf-89b8-bc38d9de1f02 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-77d927-predictor-67bbb9f4c6-cf6xf to ip-10-0-135-188.ec2.internal metadata: creationTimestamp: "2026-07-01T21:07:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:07:15Z" name: isvc-secondary-77d927-predictor-67bbb9f4c6-cf6xf.18be46841bf859e7 namespace: kserve-ci-e2e-test resourceVersion: "20222" uid: 73cf0bd3-d68b-4360-8c0c-9d66b0566419 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dc7469c8-ljvgg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:16Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-77d927-predictor-67bbb9f4c6-cf6xf namespace: kserve-ci-e2e-test resourceVersion: "20226" uid: f2881f6d-1c8b-4dcf-89b8-bc38d9de1f02 kind: Event lastTimestamp: "2026-07-01T21:07:16Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:07:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:07:16Z" name: isvc-secondary-77d927-predictor-67bbb9f4c6-cf6xf.18be46843532593b namespace: kserve-ci-e2e-test resourceVersion: "20234" uid: 5d855af4-a411-40f7-86c5-24049fad21e0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:07:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-77d927-predictor-67bbb9f4c6-cf6xf namespace: kserve-ci-e2e-test resourceVersion: "20218" uid: f2881f6d-1c8b-4dcf-89b8-bc38d9de1f02 kind: Event lastTimestamp: "2026-07-01T21:07:21Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cc7b8fac71b36ebfd1661675b86a84ade93db501973e87758605bd27f47937cb" already present on machine metadata: creationTimestamp: "2026-07-01T21:07:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:21Z" name: isvc-secondary-77d927-predictor-67bbb9f4c6-cf6xf.18be46843753f9eb namespace: kserve-ci-e2e-test resourceVersion: "20279" uid: 8306029a-4977-4399-b622-e6080deb9366 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:07:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-77d927-predictor-67bbb9f4c6-cf6xf namespace: kserve-ci-e2e-test resourceVersion: "20218" uid: f2881f6d-1c8b-4dcf-89b8-bc38d9de1f02 kind: Event lastTimestamp: "2026-07-01T21:07:21Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T21:07:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:21Z" name: isvc-secondary-77d927-predictor-67bbb9f4c6-cf6xf.18be46843a429b5f namespace: kserve-ci-e2e-test resourceVersion: "20282" uid: b70f4c87-a97a-4f4e-8d2c-5c9eceabc400 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:07:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-77d927-predictor-67bbb9f4c6-cf6xf namespace: kserve-ci-e2e-test resourceVersion: "20218" uid: f2881f6d-1c8b-4dcf-89b8-bc38d9de1f02 kind: Event lastTimestamp: "2026-07-01T21:07:21Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T21:07:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:21Z" name: isvc-secondary-77d927-predictor-67bbb9f4c6-cf6xf.18be46843ab5ce6d namespace: kserve-ci-e2e-test resourceVersion: "20283" uid: 6fd35c04-015e-4f2f-9228-8f37b98eaf29 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-77d927-predictor-67bbb9f4c6-cf6xf namespace: kserve-ci-e2e-test resourceVersion: "20218" uid: f2881f6d-1c8b-4dcf-89b8-bc38d9de1f02 kind: Event lastTimestamp: "2026-07-01T21:07:27Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-77d927-predictor-67bbb9f4c6-cf6xf_kserve-ci-e2e-test(f2881f6d-1c8b-4dcf-89b8-bc38d9de1f02) metadata: creationTimestamp: "2026-07-01T21:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:27Z" name: isvc-secondary-77d927-predictor-67bbb9f4c6-cf6xf.18be4686deaf6e3a namespace: kserve-ci-e2e-test resourceVersion: "20337" uid: bc57b57d-481a-41bb-93a2-77e3865f8fdb reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-77d927-predictor-67bbb9f4c6 namespace: kserve-ci-e2e-test resourceVersion: "20209" uid: f74842b6-a59b-413e-9d0f-0ef9aa98bc37 kind: Event lastTimestamp: "2026-07-01T21:07:15Z" message: 'Created pod: isvc-secondary-77d927-predictor-67bbb9f4c6-cf6xf' metadata: creationTimestamp: "2026-07-01T21:07:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:15Z" name: isvc-secondary-77d927-predictor-67bbb9f4c6.18be46841b8a3896 namespace: kserve-ci-e2e-test resourceVersion: "20217" uid: cc136214-f8c6-41dc-9e33-cd7b90deb8fc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-77d927-predictor namespace: kserve-ci-e2e-test resourceVersion: "20207" uid: 0499c2b6-f7ad-4c69-a918-cf317d252907 kind: Event lastTimestamp: "2026-07-01T21:07:15Z" message: Scaled up replica set isvc-secondary-77d927-predictor-67bbb9f4c6 from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:07:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:15Z" name: isvc-secondary-77d927-predictor.18be468418e13d9b namespace: kserve-ci-e2e-test resourceVersion: "20210" uid: 69cafd1c-4a11-41fe-9eaa-824d729f5bf8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-77d927-predictor namespace: kserve-ci-e2e-test resourceVersion: "20208" uid: f6859c9a-96bb-47fa-8b02-89c99ba591da kind: Event lastTimestamp: "2026-07-01T21:07:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:07:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:30Z" name: isvc-secondary-77d927-predictor.18be468797831f4f namespace: kserve-ci-e2e-test resourceVersion: "20362" uid: 69a1a6eb-7246-41f1-99b2-d31958f1d8d7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-77d927-predictor namespace: kserve-ci-e2e-test resourceVersion: "20208" uid: f6859c9a-96bb-47fa-8b02-89c99ba591da kind: Event lastTimestamp: "2026-07-01T21:07:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:07:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:30Z" name: isvc-secondary-77d927-predictor.18be468797f33dc7 namespace: kserve-ci-e2e-test resourceVersion: "20364" uid: 3842b599-41f0-42f2-b693-38a381d86c0f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-01T21:07:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-77d927 namespace: kserve-ci-e2e-test resourceVersion: "20200" uid: 1e64fa96-4953-4c77-8dd6-8e5486cafc16 kind: Event lastTimestamp: "2026-07-01T21:07:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T21:07:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:07:27Z" name: isvc-secondary-77d927.18be468418e4fffb namespace: kserve-ci-e2e-test resourceVersion: "20342" uid: aadc6e11-ce42-4c78-918b-024d83c95316 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-77d927 namespace: kserve-ci-e2e-test resourceVersion: "20200" uid: 1e64fa96-4953-4c77-8dd6-8e5486cafc16 kind: Event lastTimestamp: "2026-07-01T21:07:15Z" message: 'Failed to update status for InferenceService "isvc-secondary-77d927": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-77d927": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:07:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:07:15Z" name: isvc-secondary-77d927.18be4684195bdeb6 namespace: kserve-ci-e2e-test resourceVersion: "20213" uid: 19e61ff5-45f5-448a-a11d-a46c529d6818 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-77d927 namespace: kserve-ci-e2e-test resourceVersion: "20200" uid: 1e64fa96-4953-4c77-8dd6-8e5486cafc16 kind: Event lastTimestamp: "2026-07-01T21:07:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-77d927": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:07:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:07:15Z" name: isvc-secondary-77d927.18be4684195c1a4f namespace: kserve-ci-e2e-test resourceVersion: "20214" uid: 3699f932-d1ed-4f66-9e69-80b2fe6e422c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T20:57:44.505862Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: fffa3d25-a6b4-4f0c-bded-76ca6fa5549b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c to ip-10-0-135-188.ec2.internal metadata: creationTimestamp: "2026-07-01T20:57:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T20:57:44Z" name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c.18be45ff1de5bed7 namespace: kserve-ci-e2e-test resourceVersion: "15300" uid: a4ce39fc-4bc8-4fa6-a76e-975c5cef9d8d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dc7469c8-ljvgg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:44Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c namespace: kserve-ci-e2e-test resourceVersion: "15308" uid: fffa3d25-a6b4-4f0c-bded-76ca6fa5549b kind: Event lastTimestamp: "2026-07-01T20:57:44Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T20:57:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T20:57:44Z" name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c.18be45ff378d720c namespace: kserve-ci-e2e-test resourceVersion: "15337" uid: 384e0973-ccb2-4ffe-b70b-e71a09741bdd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: fffa3d25-a6b4-4f0c-bded-76ca6fa5549b kind: Event lastTimestamp: "2026-07-01T20:57:44Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cc7b8fac71b36ebfd1661675b86a84ade93db501973e87758605bd27f47937cb" already present on machine metadata: creationTimestamp: "2026-07-01T20:57:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:44Z" name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c.18be45ff3934fb8b namespace: kserve-ci-e2e-test resourceVersion: "15339" uid: 7b29f5b6-ac6d-4e0f-a773-1668623b0187 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: fffa3d25-a6b4-4f0c-bded-76ca6fa5549b kind: Event lastTimestamp: "2026-07-01T20:57:45Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T20:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:45Z" name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c.18be45ff3c932f11 namespace: kserve-ci-e2e-test resourceVersion: "15342" uid: f983e85e-f0a2-4f6a-b2c6-1282213efc21 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: fffa3d25-a6b4-4f0c-bded-76ca6fa5549b kind: Event lastTimestamp: "2026-07-01T20:57:45Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T20:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:45Z" name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c.18be45ff3d0ba368 namespace: kserve-ci-e2e-test resourceVersion: "15343" uid: 518d8d3b-cb13-4098-ac2b-ecf81b3ec977 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: fffa3d25-a6b4-4f0c-bded-76ca6fa5549b kind: Event lastTimestamp: "2026-07-01T20:57:49Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-01T20:57:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:49Z" name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c.18be460035fe54a1 namespace: kserve-ci-e2e-test resourceVersion: "15394" uid: 60892ab9-07bb-473f-9133-90444b8db842 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: fffa3d25-a6b4-4f0c-bded-76ca6fa5549b kind: Event lastTimestamp: "2026-07-01T20:57:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T20:57:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:49Z" name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c.18be460039a66f46 namespace: kserve-ci-e2e-test resourceVersion: "15399" uid: 00e005e8-901f-4d9a-950e-9010646dbffd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: fffa3d25-a6b4-4f0c-bded-76ca6fa5549b kind: Event lastTimestamp: "2026-07-01T20:57:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T20:57:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:49Z" name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c.18be46003a3b4624 namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: a697b7c6-c724-4d7c-ac66-235291448907 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: fffa3d25-a6b4-4f0c-bded-76ca6fa5549b kind: Event lastTimestamp: "2026-07-01T20:57:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T20:57:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:49Z" name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c.18be46003a458755 namespace: kserve-ci-e2e-test resourceVersion: "15401" uid: b5938a4e-683e-4229-ad57-c0b209daac5e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: fffa3d25-a6b4-4f0c-bded-76ca6fa5549b kind: Event lastTimestamp: "2026-07-01T20:57:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T20:57:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:49Z" name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c.18be46003d1fff3e namespace: kserve-ci-e2e-test resourceVersion: "15404" uid: 995ca411-8c6a-4b84-9c5c-e929c27618f0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: fffa3d25-a6b4-4f0c-bded-76ca6fa5549b kind: Event lastTimestamp: "2026-07-01T20:57:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:57:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:49Z" name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c.18be46003d87cbbc namespace: kserve-ci-e2e-test resourceVersion: "15405" uid: 566a4611-cab7-4e0b-860b-6c64ae450a8e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T20:57:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: fffa3d25-a6b4-4f0c-bded-76ca6fa5549b kind: Event lastTimestamp: "2026-07-01T20:59:37Z" message: 'Readiness probe failed: dial tcp 10.134.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T20:57:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:37Z" name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c.18be4600ae0aad13 namespace: kserve-ci-e2e-test resourceVersion: "16415" uid: f77f399c-a6d4-4b5c-a440-64875e3ad471 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: fffa3d25-a6b4-4f0c-bded-76ca6fa5549b kind: Event lastTimestamp: "2026-07-01T20:59:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T20:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:34Z" name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c.18be4618c9d456f0 namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: ed029603-691b-401a-9aa8-2dcc9ce55d08 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: fffa3d25-a6b4-4f0c-bded-76ca6fa5549b kind: Event lastTimestamp: "2026-07-01T20:59:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:34Z" name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c.18be4618c9d51e81 namespace: kserve-ci-e2e-test resourceVersion: "16314" uid: 29121b3f-88c6-43c4-9e79-a9fdd715d7e7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: fffa3d25-a6b4-4f0c-bded-76ca6fa5549b kind: Event lastTimestamp: "2026-07-01T20:59:37Z" message: 'Readiness probe failed: Get "https://10.134.0.20:8643/healthz": dial tcp 10.134.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T20:59:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:37Z" name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c.18be46195c6e93e2 namespace: kserve-ci-e2e-test resourceVersion: "16413" uid: a4d5f098-a053-4c15-9a58-b0df3510238b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554 namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: 51f6affd-040b-42cf-a56d-426d8dad07dd kind: Event lastTimestamp: "2026-07-01T20:57:44Z" message: 'Created pod: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554-mj72c' metadata: creationTimestamp: "2026-07-01T20:57:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:57:44Z" name: isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554.18be45ff1cea3562 namespace: kserve-ci-e2e-test resourceVersion: "15294" uid: 512ee932-e3a0-45a1-9c57-bc5a47049f6b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-c682d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15252" uid: 1cc8cc5b-6c1f-414b-999d-6e930ae08532 kind: Event lastTimestamp: "2026-07-01T20:57:44Z" message: Scaled up replica set isvc-sklearn-graph-raw-c682d-predictor-6c6fb84554 from 0 to 1 metadata: creationTimestamp: "2026-07-01T20:57:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:57:44Z" name: isvc-sklearn-graph-raw-c682d-predictor.18be45ff16ba1688 namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: 2b03d0ce-d17f-48c3-b983-cd8f9a4dd68d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T20:57:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-c682d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15254" uid: 7866d073-2bba-487b-95cb-67d05f9781e0 kind: Event lastTimestamp: "2026-07-01T20:58:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:57:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:58:29Z" name: isvc-sklearn-graph-raw-c682d-predictor.18be46029517c0cf namespace: kserve-ci-e2e-test resourceVersion: "15724" uid: 3e72b79c-6d96-4929-b93b-dc0d91e63305 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T20:57:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-c682d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15254" uid: 7866d073-2bba-487b-95cb-67d05f9781e0 kind: Event lastTimestamp: "2026-07-01T20:58:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:57:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:58:29Z" name: isvc-sklearn-graph-raw-c682d-predictor.18be4602957eeab8 namespace: kserve-ci-e2e-test resourceVersion: "15725" uid: 94a8ac50-97da-450e-bd3e-7165537a5c46 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T20:58:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-c682d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15492" uid: 7866d073-2bba-487b-95cb-67d05f9781e0 kind: Event lastTimestamp: "2026-07-01T20:59:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T20:58:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:59:29Z" name: isvc-sklearn-graph-raw-c682d-predictor.18be460d128f0a87 namespace: kserve-ci-e2e-test resourceVersion: "16207" uid: 6bddcf18-f0c2-4525-85a6-fc63898aa77a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T20:58:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-c682d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15492" uid: 7866d073-2bba-487b-95cb-67d05f9781e0 kind: Event lastTimestamp: "2026-07-01T20:59:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T20:58:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:59:29Z" name: isvc-sklearn-graph-raw-c682d-predictor.18be460d12f6fec8 namespace: kserve-ci-e2e-test resourceVersion: "16208" uid: 0d61fec2-d673-4d5b-b3c8-3628da0a23c9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T20:57:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-c682d namespace: kserve-ci-e2e-test resourceVersion: "15242" uid: 23502728-8cda-409d-acc6-6a4e757871de kind: Event lastTimestamp: "2026-07-01T20:59:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T20:57:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:59:07Z" name: isvc-sklearn-graph-raw-c682d.18be45ff16a2899a namespace: kserve-ci-e2e-test resourceVersion: "16016" uid: f3cf5f41-445e-4698-893d-3c65749c9e06 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-c682d namespace: kserve-ci-e2e-test resourceVersion: "15242" uid: 23502728-8cda-409d-acc6-6a4e757871de kind: Event lastTimestamp: "2026-07-01T20:57:44Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-c682d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-c682d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T20:57:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:57:44Z" name: isvc-sklearn-graph-raw-c682d.18be45ff1750af3c namespace: kserve-ci-e2e-test resourceVersion: "15260" uid: b3c95973-3be3-45e1-b1b0-66c457ee6f53 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-c682d namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: 23502728-8cda-409d-acc6-6a4e757871de kind: Event lastTimestamp: "2026-07-01T20:59:07Z" message: InferenceService [isvc-sklearn-graph-raw-c682d] is Ready metadata: creationTimestamp: "2026-07-01T20:59:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:59:07Z" name: isvc-sklearn-graph-raw-c682d.18be461265ae546e namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 9d42311d-ad6a-4cbf-ba89-87c2c517e9a3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T20:59:34.774981Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r namespace: kserve-ci-e2e-test resourceVersion: "16306" uid: dd4715ca-b99e-4cb8-9606-d328c0253695 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r to ip-10-0-135-188.ec2.internal metadata: creationTimestamp: "2026-07-01T20:59:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T20:59:34Z" name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r.18be4618ca7311bc namespace: kserve-ci-e2e-test resourceVersion: "16315" uid: 70be8161-9a46-44d4-a917-ddc26877beaf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dc7469c8-ljvgg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:35Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r namespace: kserve-ci-e2e-test resourceVersion: "16327" uid: dd4715ca-b99e-4cb8-9606-d328c0253695 kind: Event lastTimestamp: "2026-07-01T20:59:35Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T20:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T20:59:35Z" name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r.18be4618e449bbaf namespace: kserve-ci-e2e-test resourceVersion: "16375" uid: 0495850e-fcbd-43b7-8957-def643b2330d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: dd4715ca-b99e-4cb8-9606-d328c0253695 kind: Event lastTimestamp: "2026-07-01T20:59:35Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cc7b8fac71b36ebfd1661675b86a84ade93db501973e87758605bd27f47937cb" already present on machine metadata: creationTimestamp: "2026-07-01T20:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:35Z" name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r.18be4618e5fcf40a namespace: kserve-ci-e2e-test resourceVersion: "16378" uid: d6305c46-0b66-486d-a5e4-3790fa027c41 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: dd4715ca-b99e-4cb8-9606-d328c0253695 kind: Event lastTimestamp: "2026-07-01T20:59:35Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T20:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:35Z" name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r.18be4618e9b0f7b6 namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: 13257e16-20f0-4907-a950-fe47193ae9f9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: dd4715ca-b99e-4cb8-9606-d328c0253695 kind: Event lastTimestamp: "2026-07-01T20:59:35Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T20:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:35Z" name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r.18be4618ea2607ca namespace: kserve-ci-e2e-test resourceVersion: "16383" uid: 75b5e46f-a55a-4940-9c45-2a8d47bdfebe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: dd4715ca-b99e-4cb8-9606-d328c0253695 kind: Event lastTimestamp: "2026-07-01T20:59:39Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-01T20:59:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:39Z" name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r.18be4619eeb1cb18 namespace: kserve-ci-e2e-test resourceVersion: "16437" uid: cffa8123-5f29-43e6-9dff-f8ba73c0c5ed reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: dd4715ca-b99e-4cb8-9606-d328c0253695 kind: Event lastTimestamp: "2026-07-01T20:59:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T20:59:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:39Z" name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r.18be4619f217fd79 namespace: kserve-ci-e2e-test resourceVersion: "16443" uid: afa764b1-ba8c-413a-a48d-6bb1b6f79a56 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: dd4715ca-b99e-4cb8-9606-d328c0253695 kind: Event lastTimestamp: "2026-07-01T20:59:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T20:59:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:39Z" name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r.18be4619f29d9dbb namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: 880715eb-3094-41c0-b2ba-f056964770dd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: dd4715ca-b99e-4cb8-9606-d328c0253695 kind: Event lastTimestamp: "2026-07-01T20:59:39Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T20:59:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:39Z" name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r.18be4619f2a70860 namespace: kserve-ci-e2e-test resourceVersion: "16448" uid: c37ebe08-6204-4cc2-8085-144437ddc7f6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: dd4715ca-b99e-4cb8-9606-d328c0253695 kind: Event lastTimestamp: "2026-07-01T20:59:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T20:59:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:39Z" name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r.18be4619f57d97b9 namespace: kserve-ci-e2e-test resourceVersion: "16453" uid: dea0b2cd-e42f-4e65-a613-de7c499efccf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: dd4715ca-b99e-4cb8-9606-d328c0253695 kind: Event lastTimestamp: "2026-07-01T20:59:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:59:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:39Z" name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r.18be4619f5e6b136 namespace: kserve-ci-e2e-test resourceVersion: "16455" uid: 8842dac9-bde2-41d4-b00b-10b36b73cb48 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T20:59:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: dd4715ca-b99e-4cb8-9606-d328c0253695 kind: Event lastTimestamp: "2026-07-01T21:01:27Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T20:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:27Z" name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r.18be461a66d7433b namespace: kserve-ci-e2e-test resourceVersion: "17418" uid: ce712b21-2bc2-4696-a691-54b58b733b6a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: dd4715ca-b99e-4cb8-9606-d328c0253695 kind: Event lastTimestamp: "2026-07-01T21:01:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T21:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:25Z" name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r.18be463277515138 namespace: kserve-ci-e2e-test resourceVersion: "17357" uid: df4cfd18-e593-4c4f-8213-75689f9bb7ed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: dd4715ca-b99e-4cb8-9606-d328c0253695 kind: Event lastTimestamp: "2026-07-01T21:01:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:25Z" name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r.18be46327751c764 namespace: kserve-ci-e2e-test resourceVersion: "17359" uid: 6fa58a02-6ed4-4a72-a8ec-5e96d8195745 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: dd4715ca-b99e-4cb8-9606-d328c0253695 kind: Event lastTimestamp: "2026-07-01T21:01:27Z" message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp 10.134.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:27Z" name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r.18be463315214825 namespace: kserve-ci-e2e-test resourceVersion: "17417" uid: e036b4c9-1499-49ca-bf0b-b6109f632dfa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4 namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: da309c4a-c879-4fae-b214-509efb48113e kind: Event lastTimestamp: "2026-07-01T20:59:34Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4-bvs8r' metadata: creationTimestamp: "2026-07-01T20:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:59:34Z" name: isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4.18be4618ca0c8389 namespace: kserve-ci-e2e-test resourceVersion: "16311" uid: ea488468-d979-4c87-a49e-505608f58dc3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-1851d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16276" uid: c84a36d6-48b6-4ab2-a295-ab1f21a96561 kind: Event lastTimestamp: "2026-07-01T20:59:34Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-1851d-predictor-797cddf7c4 from 0 to 1 metadata: creationTimestamp: "2026-07-01T20:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:59:34Z" name: isvc-sklearn-graph-raw-hpa-1851d-predictor.18be4618c4e86eb0 namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: d1fc92bc-c90c-418d-a4c6-8a54fbb52309 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T20:59:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-1851d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 8bc3f5cf-cc68-482a-bba0-335c1a67001a kind: Event lastTimestamp: "2026-07-01T21:00:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:59:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:00:04Z" name: isvc-sklearn-graph-raw-hpa-1851d-predictor.18be461c43755f6f namespace: kserve-ci-e2e-test resourceVersion: "16655" uid: 67ca7bdf-6011-4bf4-b3a8-db9acf196751 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T20:59:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-1851d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 8bc3f5cf-cc68-482a-bba0-335c1a67001a kind: Event lastTimestamp: "2026-07-01T21:00:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:59:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:00:04Z" name: isvc-sklearn-graph-raw-hpa-1851d-predictor.18be461c43cbf83f namespace: kserve-ci-e2e-test resourceVersion: "16656" uid: e8cd4f29-2f16-4386-8c16-0116f65d9d44 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T21:00:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-1851d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16543" uid: 8bc3f5cf-cc68-482a-bba0-335c1a67001a kind: Event lastTimestamp: "2026-07-01T21:01:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:00:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:01:19Z" name: isvc-sklearn-graph-raw-hpa-1851d-predictor.18be462341ccc985 namespace: kserve-ci-e2e-test resourceVersion: "17250" uid: 1f6f87d1-c04d-4dae-872d-eccffe5e2afa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T21:00:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-1851d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16543" uid: 8bc3f5cf-cc68-482a-bba0-335c1a67001a kind: Event lastTimestamp: "2026-07-01T21:01:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:00:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:01:19Z" name: isvc-sklearn-graph-raw-hpa-1851d-predictor.18be46234239b84e namespace: kserve-ci-e2e-test resourceVersion: "17251" uid: 5ed5e8af-7d35-4af3-a26c-930f47840212 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T20:59:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-1851d namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: 957b3b71-b52b-40bd-bde9-fde2de25ff8e kind: Event lastTimestamp: "2026-07-01T21:00:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T20:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:00:57Z" name: isvc-sklearn-graph-raw-hpa-1851d.18be4618c5144847 namespace: kserve-ci-e2e-test resourceVersion: "17062" uid: 20fca0c8-2d0e-4262-914f-f069402c20a7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-1851d namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: 957b3b71-b52b-40bd-bde9-fde2de25ff8e kind: Event lastTimestamp: "2026-07-01T20:59:34Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-1851d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-1851d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T20:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:59:34Z" name: isvc-sklearn-graph-raw-hpa-1851d.18be4618c56baa0c namespace: kserve-ci-e2e-test resourceVersion: "16282" uid: afa6d797-38b8-4960-9205-4585dc2c3d79 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:00:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-1851d namespace: kserve-ci-e2e-test resourceVersion: "17059" uid: 957b3b71-b52b-40bd-bde9-fde2de25ff8e kind: Event lastTimestamp: "2026-07-01T21:00:57Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-1851d] is Ready metadata: creationTimestamp: "2026-07-01T21:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:00:57Z" name: isvc-sklearn-graph-raw-hpa-1851d.18be462c1e9509f4 namespace: kserve-ci-e2e-test resourceVersion: "17061" uid: 073ad7d0-d503-415d-a60c-a8732cb44267 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:03:20.363928Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2 namespace: kserve-ci-e2e-test resourceVersion: "18365" uid: db7dc66b-dfc8-48d3-8e10-024fc08f7cd6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2 to ip-10-0-135-188.ec2.internal metadata: creationTimestamp: "2026-07-01T21:03:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:03:20Z" name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2.18be464d5099298c namespace: kserve-ci-e2e-test resourceVersion: "18376" uid: 0382418c-6e18-483c-8595-480880dbdd9e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dc7469c8-ljvgg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:03:20Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2 namespace: kserve-ci-e2e-test resourceVersion: "18389" uid: db7dc66b-dfc8-48d3-8e10-024fc08f7cd6 kind: Event lastTimestamp: "2026-07-01T21:03:20Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:03:20Z" name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2.18be464d6aef8e4a namespace: kserve-ci-e2e-test resourceVersion: "18405" uid: 5f969f75-c968-4e10-8361-664e54221919 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:03:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2 namespace: kserve-ci-e2e-test resourceVersion: "18369" uid: db7dc66b-dfc8-48d3-8e10-024fc08f7cd6 kind: Event lastTimestamp: "2026-07-01T21:03:20Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cc7b8fac71b36ebfd1661675b86a84ade93db501973e87758605bd27f47937cb" already present on machine metadata: creationTimestamp: "2026-07-01T21:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:03:20Z" name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2.18be464d6c7df1a3 namespace: kserve-ci-e2e-test resourceVersion: "18407" uid: e45780b8-e336-41a2-8158-54a37834a1df reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:03:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2 namespace: kserve-ci-e2e-test resourceVersion: "18369" uid: db7dc66b-dfc8-48d3-8e10-024fc08f7cd6 kind: Event lastTimestamp: "2026-07-01T21:03:20Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T21:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:03:20Z" name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2.18be464d6f2bb91e namespace: kserve-ci-e2e-test resourceVersion: "18409" uid: 58e1d50e-f172-435c-91db-ce059d71183c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:03:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2 namespace: kserve-ci-e2e-test resourceVersion: "18369" uid: db7dc66b-dfc8-48d3-8e10-024fc08f7cd6 kind: Event lastTimestamp: "2026-07-01T21:03:20Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T21:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:03:20Z" name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2.18be464d6f918322 namespace: kserve-ci-e2e-test resourceVersion: "18410" uid: 75f866f9-8e49-40c6-94b0-826d7e93b406 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:03:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2 namespace: kserve-ci-e2e-test resourceVersion: "18369" uid: db7dc66b-dfc8-48d3-8e10-024fc08f7cd6 kind: Event lastTimestamp: "2026-07-01T21:03:24Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-01T21:03:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:03:24Z" name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2.18be464e4d88ad62 namespace: kserve-ci-e2e-test resourceVersion: "18441" uid: 50746ca2-32d7-4314-80c0-13c357417566 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:03:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2 namespace: kserve-ci-e2e-test resourceVersion: "18369" uid: db7dc66b-dfc8-48d3-8e10-024fc08f7cd6 kind: Event lastTimestamp: "2026-07-01T21:03:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T21:03:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:03:24Z" name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2.18be464e5239246f namespace: kserve-ci-e2e-test resourceVersion: "18445" uid: 1622968e-c4b8-46d3-8a56-7565323a0348 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:03:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2 namespace: kserve-ci-e2e-test resourceVersion: "18369" uid: db7dc66b-dfc8-48d3-8e10-024fc08f7cd6 kind: Event lastTimestamp: "2026-07-01T21:03:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T21:03:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:03:24Z" name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2.18be464e529ce202 namespace: kserve-ci-e2e-test resourceVersion: "18446" uid: 2be975c9-a0af-4761-bd18-e52bc37e3d36 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:03:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2 namespace: kserve-ci-e2e-test resourceVersion: "18369" uid: db7dc66b-dfc8-48d3-8e10-024fc08f7cd6 kind: Event lastTimestamp: "2026-07-01T21:03:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T21:03:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:03:24Z" name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2.18be464e52a74c9b namespace: kserve-ci-e2e-test resourceVersion: "18447" uid: 76958c99-3621-4cf1-a800-31a7d12c11e9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:03:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2 namespace: kserve-ci-e2e-test resourceVersion: "18369" uid: db7dc66b-dfc8-48d3-8e10-024fc08f7cd6 kind: Event lastTimestamp: "2026-07-01T21:03:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T21:03:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:03:24Z" name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2.18be464e555c653f namespace: kserve-ci-e2e-test resourceVersion: "18449" uid: f7dda358-edc0-4f25-a7c4-5f7fa5da3a80 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:03:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2 namespace: kserve-ci-e2e-test resourceVersion: "18369" uid: db7dc66b-dfc8-48d3-8e10-024fc08f7cd6 kind: Event lastTimestamp: "2026-07-01T21:03:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:03:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:03:24Z" name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2.18be464e55bcce7f namespace: kserve-ci-e2e-test resourceVersion: "18450" uid: 3dd3e517-3c90-4503-a885-bac7f7abbb54 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 18 eventTime: null firstTimestamp: "2026-07-01T21:03:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2 namespace: kserve-ci-e2e-test resourceVersion: "18369" uid: db7dc66b-dfc8-48d3-8e10-024fc08f7cd6 kind: Event lastTimestamp: "2026-07-01T21:05:59Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:03:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:59Z" name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2.18be464e89abf99e namespace: kserve-ci-e2e-test resourceVersion: "19628" uid: efadb51a-4149-41f0-8a8f-bf3d8b9a3e48 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2 namespace: kserve-ci-e2e-test resourceVersion: "18369" uid: db7dc66b-dfc8-48d3-8e10-024fc08f7cd6 kind: Event lastTimestamp: "2026-07-01T21:05:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T21:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:50Z" name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2.18be467043c1e1dc namespace: kserve-ci-e2e-test resourceVersion: "19517" uid: 95f1ca37-d517-4683-874d-0420e8e407bd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2 namespace: kserve-ci-e2e-test resourceVersion: "18369" uid: db7dc66b-dfc8-48d3-8e10-024fc08f7cd6 kind: Event lastTimestamp: "2026-07-01T21:05:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:05:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:50Z" name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2.18be467043c29a70 namespace: kserve-ci-e2e-test resourceVersion: "19518" uid: 5cdf7c6f-e5b6-4578-999e-9b49c832ba6c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:05:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2 namespace: kserve-ci-e2e-test resourceVersion: "18369" uid: db7dc66b-dfc8-48d3-8e10-024fc08f7cd6 kind: Event lastTimestamp: "2026-07-01T21:05:57Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:05:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:05:57Z" name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2.18be4670c40cb628 namespace: kserve-ci-e2e-test resourceVersion: "19613" uid: 95459d3b-ffdf-40cc-a822-ffa6ea2da867 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:03:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644 namespace: kserve-ci-e2e-test resourceVersion: "18340" uid: 998422a2-e2fc-4274-8bd1-27d720dbb48c kind: Event lastTimestamp: "2026-07-01T21:03:20Z" message: 'Created pod: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644-twvm2' metadata: creationTimestamp: "2026-07-01T21:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:03:20Z" name: isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644.18be464d50141de0 namespace: kserve-ci-e2e-test resourceVersion: "18368" uid: 49512f66-f390-4e5d-ac0e-747e4e3b8d8a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:03:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-a940b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18338" uid: 1c6f8cc2-d28d-4f9f-b1f0-e213793445a9 kind: Event lastTimestamp: "2026-07-01T21:03:20Z" message: Scaled up replica set isvc-sklearn-scale-raw-a940b-predictor-7fbb4f6644 from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:03:20Z" name: isvc-sklearn-scale-raw-a940b-predictor.18be464d49b9e8cd namespace: kserve-ci-e2e-test resourceVersion: "18341" uid: a86d6cd2-22ac-4a7b-8702-0b1fa7a400d2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:03:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-a940b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18339" uid: 0b23be07-50a7-4e0e-8ef9-978b7119e028 kind: Event lastTimestamp: "2026-07-01T21:03:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:03:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:03:35Z" name: isvc-sklearn-scale-raw-a940b-predictor.18be4650c8922c49 namespace: kserve-ci-e2e-test resourceVersion: "18538" uid: ded27391-531b-4c8b-a68f-2e81cf52abde reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:03:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-a940b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18339" uid: 0b23be07-50a7-4e0e-8ef9-978b7119e028 kind: Event lastTimestamp: "2026-07-01T21:03:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:03:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:03:35Z" name: isvc-sklearn-scale-raw-a940b-predictor.18be4650c8fd3ea4 namespace: kserve-ci-e2e-test resourceVersion: "18540" uid: e1646a6f-4657-47f3-84cc-a62860b97e24 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T21:03:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-a940b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18539" uid: 0b23be07-50a7-4e0e-8ef9-978b7119e028 kind: Event lastTimestamp: "2026-07-01T21:05:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:05:50Z" name: isvc-sklearn-scale-raw-a940b-predictor.18be465447aead88 namespace: kserve-ci-e2e-test resourceVersion: "19500" uid: c1e8238d-527b-4374-b19f-fb580c2c5ddd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T21:03:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-a940b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18539" uid: 0b23be07-50a7-4e0e-8ef9-978b7119e028 kind: Event lastTimestamp: "2026-07-01T21:05:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:05:50Z" name: isvc-sklearn-scale-raw-a940b-predictor.18be46544811e126 namespace: kserve-ci-e2e-test resourceVersion: "19502" uid: b35fff65-bfbc-491d-ad6e-947ef65468e7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T21:03:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-a940b namespace: kserve-ci-e2e-test resourceVersion: "18331" uid: 03dc574e-0d4d-44fa-95bb-de444ff92989 kind: Event lastTimestamp: "2026-07-01T21:05:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T21:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:05:49Z" name: isvc-sklearn-scale-raw-a940b.18be464d49b881af namespace: kserve-ci-e2e-test resourceVersion: "19493" uid: 4a7cd17f-b559-4ee4-b0f1-faf1302a6a59 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:03:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-a940b namespace: kserve-ci-e2e-test resourceVersion: "18331" uid: 03dc574e-0d4d-44fa-95bb-de444ff92989 kind: Event lastTimestamp: "2026-07-01T21:03:20Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-a940b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-a940b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:03:20Z" name: isvc-sklearn-scale-raw-a940b.18be464d4a5fa116 namespace: kserve-ci-e2e-test resourceVersion: "18345" uid: dc3784d6-7ed3-4ae4-b6dd-963dbdd5ccd0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:05:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-a940b namespace: kserve-ci-e2e-test resourceVersion: "19490" uid: 03dc574e-0d4d-44fa-95bb-de444ff92989 kind: Event lastTimestamp: "2026-07-01T21:05:49Z" message: InferenceService [isvc-sklearn-scale-raw-a940b] is Ready metadata: creationTimestamp: "2026-07-01T21:05:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:05:49Z" name: isvc-sklearn-scale-raw-a940b.18be46701370e1a8 namespace: kserve-ci-e2e-test resourceVersion: "19491" uid: 77e408e6-cd31-4789-a03e-27e8e26dd05e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T20:57:44.572731Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm namespace: kserve-ci-e2e-test resourceVersion: "15311" uid: 2e6534c9-552c-4170-a81e-d2dbcc14513c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm to ip-10-0-135-188.ec2.internal metadata: creationTimestamp: "2026-07-01T20:57:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T20:57:44Z" name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm.18be45ff21e235a1 namespace: kserve-ci-e2e-test resourceVersion: "15316" uid: da9602ee-029b-4e1d-b212-f62401726f80 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dc7469c8-ljvgg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:45Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm namespace: kserve-ci-e2e-test resourceVersion: "15322" uid: 2e6534c9-552c-4170-a81e-d2dbcc14513c kind: Event lastTimestamp: "2026-07-01T20:57:45Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T20:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T20:57:45Z" name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm.18be45ff3ba8724f namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: 2de3ccaf-e4d3-4703-b500-9aab0bc4a3de reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: 2e6534c9-552c-4170-a81e-d2dbcc14513c kind: Event lastTimestamp: "2026-07-01T20:57:45Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cc7b8fac71b36ebfd1661675b86a84ade93db501973e87758605bd27f47937cb" already present on machine metadata: creationTimestamp: "2026-07-01T20:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:45Z" name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm.18be45ff3d39e98a namespace: kserve-ci-e2e-test resourceVersion: "15344" uid: 01a3c60d-8b00-40d8-a47f-e1f5312fe226 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: 2e6534c9-552c-4170-a81e-d2dbcc14513c kind: Event lastTimestamp: "2026-07-01T20:57:45Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T20:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:45Z" name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm.18be45ff4017e8d2 namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: 8864964c-2528-446f-b02b-54820ca51f14 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: 2e6534c9-552c-4170-a81e-d2dbcc14513c kind: Event lastTimestamp: "2026-07-01T20:57:45Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T20:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:45Z" name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm.18be45ff4089df6e namespace: kserve-ci-e2e-test resourceVersion: "15346" uid: 2d9f048c-3ac4-4904-a118-435dfa92e601 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: 2e6534c9-552c-4170-a81e-d2dbcc14513c kind: Event lastTimestamp: "2026-07-01T20:57:49Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-01T20:57:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:57:49Z" name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm.18be460035eb7e3e namespace: kserve-ci-e2e-test resourceVersion: "15393" uid: f901bf49-760d-4e41-950f-eb8c0a5984ab reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:58:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: 2e6534c9-552c-4170-a81e-d2dbcc14513c kind: Event lastTimestamp: "2026-07-01T20:58:07Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 18.121s (18.121s including waiting). Image size: 1305293052 bytes.' metadata: creationTimestamp: "2026-07-01T20:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:58:07Z" name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm.18be46046e08fbf7 namespace: kserve-ci-e2e-test resourceVersion: "15553" uid: 56bd1dcd-d4c3-45fe-9d07-bd93bf9faf51 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:58:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: 2e6534c9-552c-4170-a81e-d2dbcc14513c kind: Event lastTimestamp: "2026-07-01T20:58:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T20:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:58:07Z" name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm.18be4604714a4471 namespace: kserve-ci-e2e-test resourceVersion: "15554" uid: b0f856e1-c7ea-4914-8d05-2f3f13bdb828 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:58:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: 2e6534c9-552c-4170-a81e-d2dbcc14513c kind: Event lastTimestamp: "2026-07-01T20:58:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T20:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:58:07Z" name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm.18be460471bdb807 namespace: kserve-ci-e2e-test resourceVersion: "15555" uid: 484dcbb1-b4bd-42d9-96ad-42a0efe08302 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:58:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: 2e6534c9-552c-4170-a81e-d2dbcc14513c kind: Event lastTimestamp: "2026-07-01T20:58:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T20:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:58:07Z" name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm.18be460471c7bd07 namespace: kserve-ci-e2e-test resourceVersion: "15556" uid: 9b654f1d-3af2-4cd6-838b-a258dcfbb0fe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:58:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: 2e6534c9-552c-4170-a81e-d2dbcc14513c kind: Event lastTimestamp: "2026-07-01T20:58:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T20:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:58:07Z" name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm.18be460474a21506 namespace: kserve-ci-e2e-test resourceVersion: "15558" uid: 734be73a-d2a9-4605-9782-a77e608bb883 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:58:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: 2e6534c9-552c-4170-a81e-d2dbcc14513c kind: Event lastTimestamp: "2026-07-01T20:58:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:58:07Z" name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm.18be46047513ee5a namespace: kserve-ci-e2e-test resourceVersion: "15559" uid: c096dd55-dd72-41c4-92be-64fc531c474e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-01T20:58:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: 2e6534c9-552c-4170-a81e-d2dbcc14513c kind: Event lastTimestamp: "2026-07-01T20:59:35Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T20:58:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:35Z" name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm.18be4604e3cd2137 namespace: kserve-ci-e2e-test resourceVersion: "16385" uid: e99b2c68-68b7-4527-b06c-635132c58f6e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: 2e6534c9-552c-4170-a81e-d2dbcc14513c kind: Event lastTimestamp: "2026-07-01T20:59:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T20:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:34Z" name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm.18be4618d0d3df9d namespace: kserve-ci-e2e-test resourceVersion: "16356" uid: 1e7f7941-83d5-4e98-8fbf-69cf66f9da49 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: 2e6534c9-552c-4170-a81e-d2dbcc14513c kind: Event lastTimestamp: "2026-07-01T20:59:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:34Z" name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm.18be4618d0d50597 namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: ab3f97c2-6ff0-459f-9cae-db11fe4f314b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: 2e6534c9-552c-4170-a81e-d2dbcc14513c kind: Event lastTimestamp: "2026-07-01T20:59:35Z" message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": dial tcp 10.134.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T20:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:35Z" name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm.18be4618ea010f22 namespace: kserve-ci-e2e-test resourceVersion: "16382" uid: a7853739-f5ba-4673-992f-8fee3cd50f0e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f namespace: kserve-ci-e2e-test resourceVersion: "15291" uid: 474b2385-a3d2-49c6-bd4a-671d81aadcd3 kind: Event lastTimestamp: "2026-07-01T20:57:44Z" message: 'Created pod: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f-frbjm' metadata: creationTimestamp: "2026-07-01T20:57:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:57:44Z" name: isvc-xgboost-graph-raw-c682d-predictor-6c549b746f.18be45ff2153e14f namespace: kserve-ci-e2e-test resourceVersion: "15313" uid: 1b47d39a-e717-4f25-9db2-d5bc11b6c33f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-c682d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15285" uid: 5704711e-7d20-49cf-b785-c776b61f0511 kind: Event lastTimestamp: "2026-07-01T20:57:44Z" message: Scaled up replica set isvc-xgboost-graph-raw-c682d-predictor-6c549b746f from 0 to 1 metadata: creationTimestamp: "2026-07-01T20:57:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:57:44Z" name: isvc-xgboost-graph-raw-c682d-predictor.18be45ff1d9f490a namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: 18b97254-3ad3-4255-88df-cb7438f32986 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T20:57:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-c682d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15290" uid: 55a1ee7e-8817-4011-84a9-077c63d0d223 kind: Event lastTimestamp: "2026-07-01T20:58:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:57:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:58:44Z" name: isvc-xgboost-graph-raw-c682d-predictor.18be46029be4f461 namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: 82141c5a-b033-4a09-bca3-6f29e51238e6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T20:57:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-c682d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15290" uid: 55a1ee7e-8817-4011-84a9-077c63d0d223 kind: Event lastTimestamp: "2026-07-01T20:58:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:57:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:58:44Z" name: isvc-xgboost-graph-raw-c682d-predictor.18be46029c478292 namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: ba8ee826-9e69-4879-8836-2c81175aaec9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T20:58:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-c682d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15495" uid: 55a1ee7e-8817-4011-84a9-077c63d0d223 kind: Event lastTimestamp: "2026-07-01T20:59:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T20:58:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:59:29Z" name: isvc-xgboost-graph-raw-c682d-predictor.18be46109733eedc namespace: kserve-ci-e2e-test resourceVersion: "16210" uid: ce69a7ae-7a07-4039-8431-b42a87846836 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T20:58:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-c682d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15495" uid: 55a1ee7e-8817-4011-84a9-077c63d0d223 kind: Event lastTimestamp: "2026-07-01T20:59:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T20:58:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:59:29Z" name: isvc-xgboost-graph-raw-c682d-predictor.18be4610979621cb namespace: kserve-ci-e2e-test resourceVersion: "16211" uid: 32fdd795-0dc2-4996-b7b5-6b77ab620567 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T20:57:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-c682d namespace: kserve-ci-e2e-test resourceVersion: "15262" uid: c1e0a62d-2193-4664-99d1-dc9a728c3294 kind: Event lastTimestamp: "2026-07-01T20:59:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T20:57:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:59:05Z" name: isvc-xgboost-graph-raw-c682d.18be45ff1de3d151 namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: 91cc5dd4-345c-4c4a-940e-ab1778094948 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:57:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-c682d namespace: kserve-ci-e2e-test resourceVersion: "15262" uid: c1e0a62d-2193-4664-99d1-dc9a728c3294 kind: Event lastTimestamp: "2026-07-01T20:57:44Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-c682d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-c682d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T20:57:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:57:44Z" name: isvc-xgboost-graph-raw-c682d.18be45ff1ea84123 namespace: kserve-ci-e2e-test resourceVersion: "15304" uid: 3029dd0a-b217-48d1-9b51-e6c7841581d1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-c682d namespace: kserve-ci-e2e-test resourceVersion: "15990" uid: c1e0a62d-2193-4664-99d1-dc9a728c3294 kind: Event lastTimestamp: "2026-07-01T20:59:05Z" message: InferenceService [isvc-xgboost-graph-raw-c682d] is Ready metadata: creationTimestamp: "2026-07-01T20:59:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:59:05Z" name: isvc-xgboost-graph-raw-c682d.18be4611f34842c4 namespace: kserve-ci-e2e-test resourceVersion: "15991" uid: 5783f6d3-d36d-4c35-96f7-59d0dde1d719 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T20:59:34.846883Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: dc92d55d-3972-4ca7-8d50-2fa139ab3337 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn to ip-10-0-135-188.ec2.internal metadata: creationTimestamp: "2026-07-01T20:59:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T20:59:34Z" name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn.18be4618cebc20fd namespace: kserve-ci-e2e-test resourceVersion: "16339" uid: d0d6b3c3-d4a4-4310-8510-e20204984413 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dc7469c8-ljvgg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:35Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn namespace: kserve-ci-e2e-test resourceVersion: "16355" uid: dc92d55d-3972-4ca7-8d50-2fa139ab3337 kind: Event lastTimestamp: "2026-07-01T20:59:35Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T20:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T20:59:35Z" name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn.18be4618e8c6bcc5 namespace: kserve-ci-e2e-test resourceVersion: "16379" uid: e62b72f5-9f64-44b2-a9e3-79d209903133 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: dc92d55d-3972-4ca7-8d50-2fa139ab3337 kind: Event lastTimestamp: "2026-07-01T20:59:35Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cc7b8fac71b36ebfd1661675b86a84ade93db501973e87758605bd27f47937cb" already present on machine metadata: creationTimestamp: "2026-07-01T20:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:35Z" name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn.18be4618ea297b04 namespace: kserve-ci-e2e-test resourceVersion: "16384" uid: 6d850493-8c90-487b-8af5-4f99819e9854 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: dc92d55d-3972-4ca7-8d50-2fa139ab3337 kind: Event lastTimestamp: "2026-07-01T20:59:35Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T20:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:35Z" name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn.18be4618ec70532f namespace: kserve-ci-e2e-test resourceVersion: "16386" uid: e0e67581-a7ca-4b55-9f14-857dfd840923 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: dc92d55d-3972-4ca7-8d50-2fa139ab3337 kind: Event lastTimestamp: "2026-07-01T20:59:35Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T20:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:35Z" name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn.18be4618ecfca8bf namespace: kserve-ci-e2e-test resourceVersion: "16387" uid: 9327f755-33cb-4a5f-8f13-10dc6b8e74f3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: dc92d55d-3972-4ca7-8d50-2fa139ab3337 kind: Event lastTimestamp: "2026-07-01T20:59:39Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-07-01T20:59:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:39Z" name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn.18be4619ee9c1736 namespace: kserve-ci-e2e-test resourceVersion: "16436" uid: e2793aec-4094-4f18-8baa-6702e5c95170 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: dc92d55d-3972-4ca7-8d50-2fa139ab3337 kind: Event lastTimestamp: "2026-07-01T20:59:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T20:59:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:39Z" name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn.18be4619f22ffe66 namespace: kserve-ci-e2e-test resourceVersion: "16446" uid: a2bc935f-dec4-45f1-9db7-7ee188c9e8cb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: dc92d55d-3972-4ca7-8d50-2fa139ab3337 kind: Event lastTimestamp: "2026-07-01T20:59:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T20:59:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:39Z" name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn.18be4619f2cb061b namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: bfc14a19-74b1-4bc4-a0a9-db5b3ee3c55a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: dc92d55d-3972-4ca7-8d50-2fa139ab3337 kind: Event lastTimestamp: "2026-07-01T20:59:39Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T20:59:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:39Z" name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn.18be4619f2d30f2c namespace: kserve-ci-e2e-test resourceVersion: "16450" uid: 2e6dee69-380a-4280-b421-c3ba10db0082 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: dc92d55d-3972-4ca7-8d50-2fa139ab3337 kind: Event lastTimestamp: "2026-07-01T20:59:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T20:59:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:39Z" name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn.18be4619f5931c6b namespace: kserve-ci-e2e-test resourceVersion: "16454" uid: 58aea22c-f06b-4d85-9e0a-a3365b95beab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: dc92d55d-3972-4ca7-8d50-2fa139ab3337 kind: Event lastTimestamp: "2026-07-01T20:59:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T20:59:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:39Z" name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn.18be4619f5f8bb43 namespace: kserve-ci-e2e-test resourceVersion: "16456" uid: 1c1f3db4-2862-4872-9204-a5361f2da852 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-01T20:59:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: dc92d55d-3972-4ca7-8d50-2fa139ab3337 kind: Event lastTimestamp: "2026-07-01T21:01:26Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T20:59:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:26Z" name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn.18be461a2adbf0f6 namespace: kserve-ci-e2e-test resourceVersion: "17400" uid: 561f16e5-2ad6-4445-9cb6-30e8b88f890f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: dc92d55d-3972-4ca7-8d50-2fa139ab3337 kind: Event lastTimestamp: "2026-07-01T21:01:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T21:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:25Z" name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn.18be463279cc334d namespace: kserve-ci-e2e-test resourceVersion: "17377" uid: d2e90d46-e012-408f-88d1-6213b69233db reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: dc92d55d-3972-4ca7-8d50-2fa139ab3337 kind: Event lastTimestamp: "2026-07-01T21:01:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:25Z" name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn.18be463279ccb607 namespace: kserve-ci-e2e-test resourceVersion: "17378" uid: c838c728-ca75-4e08-bdb7-9de49b8203c0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: dc92d55d-3972-4ca7-8d50-2fa139ab3337 kind: Event lastTimestamp: "2026-07-01T21:01:26Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:26Z" name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn.18be4632d9427888 namespace: kserve-ci-e2e-test resourceVersion: "17399" uid: ea3dca8d-1b62-4f1b-b58f-89de4add0644 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957 namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: 0d609933-7598-4317-9ebb-76b74ba48954 kind: Event lastTimestamp: "2026-07-01T20:59:34Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957-4tcvn' metadata: creationTimestamp: "2026-07-01T20:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:59:34Z" name: isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957.18be4618ce4b1d5e namespace: kserve-ci-e2e-test resourceVersion: "16336" uid: a8f9960b-ef6d-4734-9d82-7439ae8b08af reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-1851d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: 6845beda-6cf0-47a8-ba8c-6bbf871143f5 kind: Event lastTimestamp: "2026-07-01T20:59:34Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-1851d-predictor-6cf48c7957 from 0 to 1 metadata: creationTimestamp: "2026-07-01T20:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:59:34Z" name: isvc-xgboost-graph-raw-hpa-1851d-predictor.18be4618cb261464 namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: f79a8bef-b7fb-40b7-af35-5488575d2104 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T20:59:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-1851d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16316" uid: 952222ea-ad49-4d9a-a999-bedd6bdb3991 kind: Event lastTimestamp: "2026-07-01T21:00:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:59:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:00:04Z" name: isvc-xgboost-graph-raw-hpa-1851d-predictor.18be461c49a71e60 namespace: kserve-ci-e2e-test resourceVersion: "16659" uid: a308484f-96ff-45b6-b737-8085c211ab0c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T20:59:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-1851d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16316" uid: 952222ea-ad49-4d9a-a999-bedd6bdb3991 kind: Event lastTimestamp: "2026-07-01T21:00:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T20:59:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:00:04Z" name: isvc-xgboost-graph-raw-hpa-1851d-predictor.18be461c4a0ffdb8 namespace: kserve-ci-e2e-test resourceVersion: "16660" uid: 7e905b79-9606-40a2-a11c-2cf154d6df2d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:00:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-1851d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 952222ea-ad49-4d9a-a999-bedd6bdb3991 kind: Event lastTimestamp: "2026-07-01T21:00:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:00:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:00:49Z" name: isvc-xgboost-graph-raw-hpa-1851d-predictor.18be4623479c14d8 namespace: kserve-ci-e2e-test resourceVersion: "16994" uid: 3f20c0b0-cf86-42ff-a449-bdff1284df80 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:00:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-1851d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 952222ea-ad49-4d9a-a999-bedd6bdb3991 kind: Event lastTimestamp: "2026-07-01T21:00:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:00:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:00:49Z" name: isvc-xgboost-graph-raw-hpa-1851d-predictor.18be462348108c66 namespace: kserve-ci-e2e-test resourceVersion: "16995" uid: 0c4c02f7-5cc6-47b5-9f37-a72677f57fbf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T20:59:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-1851d namespace: kserve-ci-e2e-test resourceVersion: "16284" uid: 7aa5f138-7376-4bee-8114-980948d71b57 kind: Event lastTimestamp: "2026-07-01T21:00:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T20:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:00:36Z" name: isvc-xgboost-graph-raw-hpa-1851d.18be4618cb664da1 namespace: kserve-ci-e2e-test resourceVersion: "16907" uid: 8dcc48c0-adbe-4d85-8dec-0c82eb35570b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-1851d namespace: kserve-ci-e2e-test resourceVersion: "16284" uid: 7aa5f138-7376-4bee-8114-980948d71b57 kind: Event lastTimestamp: "2026-07-01T20:59:34Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-hpa-1851d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-hpa-1851d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T20:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:59:34Z" name: isvc-xgboost-graph-raw-hpa-1851d.18be4618cbfa5f66 namespace: kserve-ci-e2e-test resourceVersion: "16326" uid: 1b719b63-95c5-432c-aaa2-803ace9bd90c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:00:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-1851d namespace: kserve-ci-e2e-test resourceVersion: "16904" uid: 7aa5f138-7376-4bee-8114-980948d71b57 kind: Event lastTimestamp: "2026-07-01T21:00:36Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-1851d] is Ready metadata: creationTimestamp: "2026-07-01T21:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:00:36Z" name: isvc-xgboost-graph-raw-hpa-1851d.18be46273a83873a namespace: kserve-ci-e2e-test resourceVersion: "16905" uid: 97cf2f8c-1021-489a-a9c4-5bdce3178aaa reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:01:25.000475Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-1255c-predictor-cc944bc4-sxm5p namespace: kserve-ci-e2e-test resourceVersion: "17332" uid: 97566295-827d-4bd2-8a44-231e50c2bce0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-1255c-predictor-cc944bc4-sxm5p to ip-10-0-135-188.ec2.internal metadata: creationTimestamp: "2026-07-01T21:01:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:01:25Z" name: message-dumper-raw-1255c-predictor-cc944bc4-sxm5p.18be4632746693f1 namespace: kserve-ci-e2e-test resourceVersion: "17337" uid: cc9c5c6f-e069-4e7b-aec8-6ca3016afa2c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dc7469c8-ljvgg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:25Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-1255c-predictor-cc944bc4-sxm5p namespace: kserve-ci-e2e-test resourceVersion: "17334" uid: 97566295-827d-4bd2-8a44-231e50c2bce0 kind: Event lastTimestamp: "2026-07-01T21:01:25Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "message-dumper-raw-1255c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T21:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:25Z" name: message-dumper-raw-1255c-predictor-cc944bc4-sxm5p.18be46328444ac09 namespace: kserve-ci-e2e-test resourceVersion: "17384" uid: d80da1b4-dc30-4c95-856a-2dbe0495ff28 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:26Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-1255c-predictor-cc944bc4-sxm5p namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: 97566295-827d-4bd2-8a44-231e50c2bce0 kind: Event lastTimestamp: "2026-07-01T21:01:26Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:01:26Z" name: message-dumper-raw-1255c-predictor-cc944bc4-sxm5p.18be4632b18d2ff9 namespace: kserve-ci-e2e-test resourceVersion: "17393" uid: 0f5e23f6-9cbc-4e35-bfc9-e8b3621b6cce reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-1255c-predictor-cc944bc4-sxm5p namespace: kserve-ci-e2e-test resourceVersion: "17334" uid: 97566295-827d-4bd2-8a44-231e50c2bce0 kind: Event lastTimestamp: "2026-07-01T21:01:26Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-01T21:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:26Z" name: message-dumper-raw-1255c-predictor-cc944bc4-sxm5p.18be4632b2f2f461 namespace: kserve-ci-e2e-test resourceVersion: "17395" uid: 666a6aa6-7092-4279-ade1-b074705a4255 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-1255c-predictor-cc944bc4-sxm5p namespace: kserve-ci-e2e-test resourceVersion: "17334" uid: 97566295-827d-4bd2-8a44-231e50c2bce0 kind: Event lastTimestamp: "2026-07-01T21:01:27Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.175s (1.175s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-01T21:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:27Z" name: message-dumper-raw-1255c-predictor-cc944bc4-sxm5p.18be4632f908191d namespace: kserve-ci-e2e-test resourceVersion: "17408" uid: 12454a01-0f8a-4e8c-aa20-199f5b1c19b4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-1255c-predictor-cc944bc4-sxm5p namespace: kserve-ci-e2e-test resourceVersion: "17334" uid: 97566295-827d-4bd2-8a44-231e50c2bce0 kind: Event lastTimestamp: "2026-07-01T21:01:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T21:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:27Z" name: message-dumper-raw-1255c-predictor-cc944bc4-sxm5p.18be4632fbd92b71 namespace: kserve-ci-e2e-test resourceVersion: "17409" uid: 26cd3f22-806a-46e9-97c3-994e285878d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-1255c-predictor-cc944bc4-sxm5p namespace: kserve-ci-e2e-test resourceVersion: "17334" uid: 97566295-827d-4bd2-8a44-231e50c2bce0 kind: Event lastTimestamp: "2026-07-01T21:01:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T21:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:27Z" name: message-dumper-raw-1255c-predictor-cc944bc4-sxm5p.18be4632fc4e6ac0 namespace: kserve-ci-e2e-test resourceVersion: "17410" uid: cb99d9cc-4172-43e5-9e3b-5b93e75c6c89 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-1255c-predictor-cc944bc4-sxm5p namespace: kserve-ci-e2e-test resourceVersion: "17334" uid: 97566295-827d-4bd2-8a44-231e50c2bce0 kind: Event lastTimestamp: "2026-07-01T21:01:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T21:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:27Z" name: message-dumper-raw-1255c-predictor-cc944bc4-sxm5p.18be4632fc58114d namespace: kserve-ci-e2e-test resourceVersion: "17411" uid: b4cb32d3-5c1d-41f8-bbfd-bcef7e8c6ccf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-1255c-predictor-cc944bc4-sxm5p namespace: kserve-ci-e2e-test resourceVersion: "17334" uid: 97566295-827d-4bd2-8a44-231e50c2bce0 kind: Event lastTimestamp: "2026-07-01T21:01:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T21:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:27Z" name: message-dumper-raw-1255c-predictor-cc944bc4-sxm5p.18be4632ffeb561e namespace: kserve-ci-e2e-test resourceVersion: "17412" uid: b40e573c-513c-4bf5-8e83-46250b5c1099 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-1255c-predictor-cc944bc4-sxm5p namespace: kserve-ci-e2e-test resourceVersion: "17334" uid: 97566295-827d-4bd2-8a44-231e50c2bce0 kind: Event lastTimestamp: "2026-07-01T21:01:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:27Z" name: message-dumper-raw-1255c-predictor-cc944bc4-sxm5p.18be46330061bef7 namespace: kserve-ci-e2e-test resourceVersion: "17413" uid: 66ce2fce-4124-4aed-8b59-767b752e5d29 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:03:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-1255c-predictor-cc944bc4-sxm5p namespace: kserve-ci-e2e-test resourceVersion: "17334" uid: 97566295-827d-4bd2-8a44-231e50c2bce0 kind: Event lastTimestamp: "2026-07-01T21:03:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T21:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:03:20Z" name: message-dumper-raw-1255c-predictor-cc944bc4-sxm5p.18be464d5321753a namespace: kserve-ci-e2e-test resourceVersion: "18390" uid: bd8fab77-0bc2-4bcb-8322-c2718f5839e0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:03:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-1255c-predictor-cc944bc4-sxm5p namespace: kserve-ci-e2e-test resourceVersion: "17334" uid: 97566295-827d-4bd2-8a44-231e50c2bce0 kind: Event lastTimestamp: "2026-07-01T21:03:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:03:20Z" name: message-dumper-raw-1255c-predictor-cc944bc4-sxm5p.18be464d532217d4 namespace: kserve-ci-e2e-test resourceVersion: "18393" uid: 2e83d6b3-3dca-4d1f-847d-12616cf0554a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-1255c-predictor-cc944bc4 namespace: kserve-ci-e2e-test resourceVersion: "17321" uid: 54476589-f0a1-4e68-b436-c44eec7751e7 kind: Event lastTimestamp: "2026-07-01T21:01:24Z" message: 'Created pod: message-dumper-raw-1255c-predictor-cc944bc4-sxm5p' metadata: creationTimestamp: "2026-07-01T21:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:01:24Z" name: message-dumper-raw-1255c-predictor-cc944bc4.18be463273dcb057 namespace: kserve-ci-e2e-test resourceVersion: "17333" uid: 0d8b7dcc-5c87-46fe-8fd4-635900b464ad reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-1255c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17319" uid: e35672d7-69cf-4a54-9637-84447fea7b9a kind: Event lastTimestamp: "2026-07-01T21:01:24Z" message: Scaled up replica set message-dumper-raw-1255c-predictor-cc944bc4 from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:01:24Z" name: message-dumper-raw-1255c-predictor.18be463270bb1815 namespace: kserve-ci-e2e-test resourceVersion: "17323" uid: 58b353ce-869d-4fcb-836a-7371aee4caed reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:01:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-1255c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17320" uid: 7c29eeba-e28a-4875-a868-8a1fc3b400b1 kind: Event lastTimestamp: "2026-07-01T21:01:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:01:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:01:54Z" name: message-dumper-raw-1255c-predictor.18be4635efa1cdc2 namespace: kserve-ci-e2e-test resourceVersion: "17699" uid: 50d176f6-cb81-454c-8320-704ba4274f99 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:01:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-1255c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17320" uid: 7c29eeba-e28a-4875-a868-8a1fc3b400b1 kind: Event lastTimestamp: "2026-07-01T21:01:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:01:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:01:54Z" name: message-dumper-raw-1255c-predictor.18be4635effead73 namespace: kserve-ci-e2e-test resourceVersion: "17700" uid: 27daa5a6-16e2-4d14-ab00-1f4e434e8f13 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T21:01:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-1255c namespace: kserve-ci-e2e-test resourceVersion: "17312" uid: 874ee16c-43c1-4c4d-b14c-4921bb6442a5 kind: Event lastTimestamp: "2026-07-01T21:01:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T21:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:01:35Z" name: message-dumper-raw-1255c.18be463270b98972 namespace: kserve-ci-e2e-test resourceVersion: "17487" uid: 516566c3-c528-47db-9edb-18364f8965ba reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-1255c namespace: kserve-ci-e2e-test resourceVersion: "17312" uid: 874ee16c-43c1-4c4d-b14c-4921bb6442a5 kind: Event lastTimestamp: "2026-07-01T21:01:24Z" message: 'Failed to update status for InferenceService "message-dumper-raw-1255c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-1255c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:01:24Z" name: message-dumper-raw-1255c.18be463271085831 namespace: kserve-ci-e2e-test resourceVersion: "17326" uid: 34c85d35-b588-42dd-ae17-672617fed4f7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-1255c namespace: kserve-ci-e2e-test resourceVersion: "17485" uid: 874ee16c-43c1-4c4d-b14c-4921bb6442a5 kind: Event lastTimestamp: "2026-07-01T21:01:35Z" message: InferenceService [message-dumper-raw-1255c] is Ready metadata: creationTimestamp: "2026-07-01T21:01:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:01:35Z" name: message-dumper-raw-1255c.18be4634d536a78f namespace: kserve-ci-e2e-test resourceVersion: "17486" uid: 73cf845c-9d08-4f49-86d3-d33821b42bec reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T20:59:24.463452Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-c682d-544fdc8c65-64hj5 namespace: kserve-ci-e2e-test resourceVersion: "16147" uid: 7d2ec01c-ca66-4cd5-8379-5fcd3d9da30d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-c682d-544fdc8c65-64hj5 to ip-10-0-137-220.ec2.internal metadata: creationTimestamp: "2026-07-01T20:59:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T20:59:24Z" name: model-chainer-raw-c682d-544fdc8c65-64hj5.18be461663d58ab7 namespace: kserve-ci-e2e-test resourceVersion: "16153" uid: 48377b38-e03e-4ca1-b4c7-8c0412e24d1c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dc7469c8-ljvgg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:24Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-c682d-544fdc8c65-64hj5 namespace: kserve-ci-e2e-test resourceVersion: "16151" uid: 7d2ec01c-ca66-4cd5-8379-5fcd3d9da30d kind: Event lastTimestamp: "2026-07-01T20:59:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-c682d-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T20:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:24Z" name: model-chainer-raw-c682d-544fdc8c65-64hj5.18be461670461b3d namespace: kserve-ci-e2e-test resourceVersion: "16164" uid: 886b39ec-9c63-4378-b21b-8b901e5ddbc8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-220.ec2.internal source: component: kubelet host: ip-10-0-137-220.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:25Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-c682d-544fdc8c65-64hj5 namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 7d2ec01c-ca66-4cd5-8379-5fcd3d9da30d kind: Event lastTimestamp: "2026-07-01T20:59:25Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T20:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T20:59:25Z" name: model-chainer-raw-c682d-544fdc8c65-64hj5.18be4616a104be71 namespace: kserve-ci-e2e-test resourceVersion: "16172" uid: 1ce0f086-9762-4e98-8e1c-714c1f9c4688 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c682d} kind: Pod name: model-chainer-raw-c682d-544fdc8c65-64hj5 namespace: kserve-ci-e2e-test resourceVersion: "16151" uid: 7d2ec01c-ca66-4cd5-8379-5fcd3d9da30d kind: Event lastTimestamp: "2026-07-01T20:59:25Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:78561fce36825084e3c48d0137c513c345701cffde73e206abb14ec219f8181b" metadata: creationTimestamp: "2026-07-01T20:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:25Z" name: model-chainer-raw-c682d-544fdc8c65-64hj5.18be4616a247b7fb namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: b9f562eb-f571-4c7a-8399-81b935fcc3ec reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-220.ec2.internal source: component: kubelet host: ip-10-0-137-220.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c682d} kind: Pod name: model-chainer-raw-c682d-544fdc8c65-64hj5 namespace: kserve-ci-e2e-test resourceVersion: "16151" uid: 7d2ec01c-ca66-4cd5-8379-5fcd3d9da30d kind: Event lastTimestamp: "2026-07-01T20:59:28Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:78561fce36825084e3c48d0137c513c345701cffde73e206abb14ec219f8181b" in 2.699s (2.699s including waiting). Image size: 218070105 bytes.' metadata: creationTimestamp: "2026-07-01T20:59:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:28Z" name: model-chainer-raw-c682d-544fdc8c65-64hj5.18be46174333a2bf namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: 93003ba0-a7de-4bea-9f0d-58911a723622 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-220.ec2.internal source: component: kubelet host: ip-10-0-137-220.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c682d} kind: Pod name: model-chainer-raw-c682d-544fdc8c65-64hj5 namespace: kserve-ci-e2e-test resourceVersion: "16151" uid: 7d2ec01c-ca66-4cd5-8379-5fcd3d9da30d kind: Event lastTimestamp: "2026-07-01T20:59:28Z" message: 'Created container: model-chainer-raw-c682d' metadata: creationTimestamp: "2026-07-01T20:59:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:28Z" name: model-chainer-raw-c682d-544fdc8c65-64hj5.18be4617466d4eff namespace: kserve-ci-e2e-test resourceVersion: "16195" uid: 11aa6ed0-832c-4991-be6f-c7eebe39b5b2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-220.ec2.internal source: component: kubelet host: ip-10-0-137-220.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c682d} kind: Pod name: model-chainer-raw-c682d-544fdc8c65-64hj5 namespace: kserve-ci-e2e-test resourceVersion: "16151" uid: 7d2ec01c-ca66-4cd5-8379-5fcd3d9da30d kind: Event lastTimestamp: "2026-07-01T20:59:28Z" message: Started container model-chainer-raw-c682d metadata: creationTimestamp: "2026-07-01T20:59:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:28Z" name: model-chainer-raw-c682d-544fdc8c65-64hj5.18be461746f1492d namespace: kserve-ci-e2e-test resourceVersion: "16196" uid: 50679569-cea8-4729-925a-952d80497788 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-220.ec2.internal source: component: kubelet host: ip-10-0-137-220.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c682d} kind: Pod name: model-chainer-raw-c682d-544fdc8c65-64hj5 namespace: kserve-ci-e2e-test resourceVersion: "16151" uid: 7d2ec01c-ca66-4cd5-8379-5fcd3d9da30d kind: Event lastTimestamp: "2026-07-01T20:59:34Z" message: Stopping container model-chainer-raw-c682d metadata: creationTimestamp: "2026-07-01T20:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T20:59:34Z" name: model-chainer-raw-c682d-544fdc8c65-64hj5.18be4618bad15088 namespace: kserve-ci-e2e-test resourceVersion: "16259" uid: 7fd02ec8-1d92-45be-9ba6-e5479e762aab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-220.ec2.internal source: component: kubelet host: ip-10-0-137-220.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T20:59:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-c682d} kind: Pod name: model-chainer-raw-c682d-544fdc8c65-64hj5 namespace: kserve-ci-e2e-test resourceVersion: "16151" uid: 7d2ec01c-ca66-4cd5-8379-5fcd3d9da30d kind: Event lastTimestamp: "2026-07-01T21:00:04Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T20:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:00:04Z" name: model-chainer-raw-c682d-544fdc8c65-64hj5.18be4618bb2bd9a7 namespace: kserve-ci-e2e-test resourceVersion: "16654" uid: a0609253-d01a-49a0-ab64-f3a5cc8c348b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-220.ec2.internal source: component: kubelet host: ip-10-0-137-220.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-c682d-544fdc8c65 namespace: kserve-ci-e2e-test resourceVersion: "16144" uid: 7d03f4c2-01a9-42a0-b5c3-4e4688f4d31b kind: Event lastTimestamp: "2026-07-01T20:59:24Z" message: 'Created pod: model-chainer-raw-c682d-544fdc8c65-64hj5' metadata: creationTimestamp: "2026-07-01T20:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:59:24Z" name: model-chainer-raw-c682d-544fdc8c65.18be46166357be3c namespace: kserve-ci-e2e-test resourceVersion: "16150" uid: 093f7b07-ae04-41c0-abae-ed1ea1d4ad80 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-c682d namespace: kserve-ci-e2e-test resourceVersion: "16142" uid: 6d61523a-9957-49f4-8309-6d72f42f1bbe kind: Event lastTimestamp: "2026-07-01T20:59:24Z" message: Scaled up replica set model-chainer-raw-c682d-544fdc8c65 from 0 to 1 metadata: creationTimestamp: "2026-07-01T20:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T20:59:24Z" name: model-chainer-raw-c682d.18be461662427d97 namespace: kserve-ci-e2e-test resourceVersion: "16145" uid: a0f4d2df-8ff0-4869-a01c-9a5918294972 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T20:59:24Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-c682d namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: a2bcb28c-9c88-4e66-a3f2-58eef1ba33c3 kind: Event lastTimestamp: "2026-07-01T20:59:24Z" message: InferenceGraph [model-chainer-raw-c682d] is Ready metadata: creationTimestamp: "2026-07-01T20:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T20:59:24Z" name: model-chainer-raw-c682d.18be46166746aa6a namespace: kserve-ci-e2e-test resourceVersion: "16161" uid: 8bf2bca0-a167-43cc-ac4e-c617a1256c0c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:01:14.805087Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-1851d-c96567755-bzhlm namespace: kserve-ci-e2e-test resourceVersion: "17190" uid: 7247b698-fab8-49ed-ae96-ae7bc0bb1296 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-1851d-c96567755-bzhlm to ip-10-0-137-220.ec2.internal metadata: creationTimestamp: "2026-07-01T21:01:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:01:14Z" name: model-chainer-raw-hpa-1851d-c96567755-bzhlm.18be463014b54ba8 namespace: kserve-ci-e2e-test resourceVersion: "17196" uid: 54a2b5e3-5488-4dfa-beed-bfcbfa28809d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dc7469c8-ljvgg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:15Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-1851d-c96567755-bzhlm namespace: kserve-ci-e2e-test resourceVersion: "17199" uid: 7247b698-fab8-49ed-ae96-ae7bc0bb1296 kind: Event lastTimestamp: "2026-07-01T21:01:15Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:01:15Z" name: model-chainer-raw-hpa-1851d-c96567755-bzhlm.18be46302e6708d2 namespace: kserve-ci-e2e-test resourceVersion: "17210" uid: b9c54ded-e641-41b0-b4d8-5a0f9fcf3b72 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-1851d} kind: Pod name: model-chainer-raw-hpa-1851d-c96567755-bzhlm namespace: kserve-ci-e2e-test resourceVersion: "17194" uid: 7247b698-fab8-49ed-ae96-ae7bc0bb1296 kind: Event lastTimestamp: "2026-07-01T21:01:15Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:78561fce36825084e3c48d0137c513c345701cffde73e206abb14ec219f8181b" already present on machine metadata: creationTimestamp: "2026-07-01T21:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:15Z" name: model-chainer-raw-hpa-1851d-c96567755-bzhlm.18be46302fb2d4de namespace: kserve-ci-e2e-test resourceVersion: "17212" uid: 97c49646-3b1a-4519-a9c0-0d10c70fdef8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-220.ec2.internal source: component: kubelet host: ip-10-0-137-220.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-1851d} kind: Pod name: model-chainer-raw-hpa-1851d-c96567755-bzhlm namespace: kserve-ci-e2e-test resourceVersion: "17194" uid: 7247b698-fab8-49ed-ae96-ae7bc0bb1296 kind: Event lastTimestamp: "2026-07-01T21:01:15Z" message: 'Created container: model-chainer-raw-hpa-1851d' metadata: creationTimestamp: "2026-07-01T21:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:15Z" name: model-chainer-raw-hpa-1851d-c96567755-bzhlm.18be463032ba3c12 namespace: kserve-ci-e2e-test resourceVersion: "17213" uid: fae2f66c-4769-4c73-bc94-bc06cc927768 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-220.ec2.internal source: component: kubelet host: ip-10-0-137-220.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-1851d} kind: Pod name: model-chainer-raw-hpa-1851d-c96567755-bzhlm namespace: kserve-ci-e2e-test resourceVersion: "17194" uid: 7247b698-fab8-49ed-ae96-ae7bc0bb1296 kind: Event lastTimestamp: "2026-07-01T21:01:15Z" message: Started container model-chainer-raw-hpa-1851d metadata: creationTimestamp: "2026-07-01T21:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:15Z" name: model-chainer-raw-hpa-1851d-c96567755-bzhlm.18be463033273db9 namespace: kserve-ci-e2e-test resourceVersion: "17214" uid: 1814c4e7-e1d7-4b8b-8f1a-3cffb87984fc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-220.ec2.internal source: component: kubelet host: ip-10-0-137-220.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-1851d} kind: Pod name: model-chainer-raw-hpa-1851d-c96567755-bzhlm namespace: kserve-ci-e2e-test resourceVersion: "17194" uid: 7247b698-fab8-49ed-ae96-ae7bc0bb1296 kind: Event lastTimestamp: "2026-07-01T21:01:24Z" message: Stopping container model-chainer-raw-hpa-1851d metadata: creationTimestamp: "2026-07-01T21:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:24Z" name: model-chainer-raw-hpa-1851d-c96567755-bzhlm.18be463268bb6f33 namespace: kserve-ci-e2e-test resourceVersion: "17307" uid: 12b116c9-9b51-4de5-9266-c91199afb6fd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-220.ec2.internal source: component: kubelet host: ip-10-0-137-220.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T21:01:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-1851d} kind: Pod name: model-chainer-raw-hpa-1851d-c96567755-bzhlm namespace: kserve-ci-e2e-test resourceVersion: "17194" uid: 7247b698-fab8-49ed-ae96-ae7bc0bb1296 kind: Event lastTimestamp: "2026-07-01T21:01:52Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T21:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:01:52Z" name: model-chainer-raw-hpa-1851d-c96567755-bzhlm.18be4632f7b2b009 namespace: kserve-ci-e2e-test resourceVersion: "17678" uid: eb21b4a1-3dd9-49a9-aece-d8672486f241 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-220.ec2.internal source: component: kubelet host: ip-10-0-137-220.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-1851d-c96567755 namespace: kserve-ci-e2e-test resourceVersion: "17185" uid: 882aa5bc-1b63-4816-b6c7-83304baede64 kind: Event lastTimestamp: "2026-07-01T21:01:14Z" message: 'Created pod: model-chainer-raw-hpa-1851d-c96567755-bzhlm' metadata: creationTimestamp: "2026-07-01T21:01:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:01:14Z" name: model-chainer-raw-hpa-1851d-c96567755.18be463013cddbf9 namespace: kserve-ci-e2e-test resourceVersion: "17191" uid: 0a604691-af04-4bc5-ab09-62323e71a4e6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-1851d namespace: kserve-ci-e2e-test resourceVersion: "17184" uid: 237b9098-27cb-4eb4-a2d1-472ff98e0528 kind: Event lastTimestamp: "2026-07-01T21:01:14Z" message: Scaled up replica set model-chainer-raw-hpa-1851d-c96567755 from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:01:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:01:14Z" name: model-chainer-raw-hpa-1851d.18be4630123dbc0f namespace: kserve-ci-e2e-test resourceVersion: "17188" uid: 9efaa19d-7961-4098-9c6c-2f5a1737eca1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:01:14Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-1851d namespace: kserve-ci-e2e-test resourceVersion: "17201" uid: 03404410-ab38-4fc5-85f0-39edc4059bf1 kind: Event lastTimestamp: "2026-07-01T21:01:14Z" message: InferenceGraph [model-chainer-raw-hpa-1851d] is Ready metadata: creationTimestamp: "2026-07-01T21:01:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:01:14Z" name: model-chainer-raw-hpa-1851d.18be463019629b32 namespace: kserve-ci-e2e-test resourceVersion: "17203" uid: 882cdbbb-5f5b-40d8-af91-8fdf9ba7ccad reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:07:54.274946Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs namespace: kserve-ci-e2e-test resourceVersion: "20675" uid: d3ebb858-b3ad-44ca-9ee6-ef6508714873 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-92cf8-predictor-d4757fc6b-s49vs to ip-10-0-135-188.ec2.internal metadata: creationTimestamp: "2026-07-01T21:07:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:07:54Z" name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs.18be468d16f7ae04 namespace: kserve-ci-e2e-test resourceVersion: "20680" uid: f60ccd98-f590-42ad-adc3-b6c4bf7bf817 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dc7469c8-ljvgg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:54Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: d3ebb858-b3ad-44ca-9ee6-ef6508714873 kind: Event lastTimestamp: "2026-07-01T21:07:54Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-92cf8-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T21:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:54Z" name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs.18be468d217989f0 namespace: kserve-ci-e2e-test resourceVersion: "20689" uid: a513f13c-4848-48f5-8327-ab9a08d1fea7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:55Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs namespace: kserve-ci-e2e-test resourceVersion: "20684" uid: d3ebb858-b3ad-44ca-9ee6-ef6508714873 kind: Event lastTimestamp: "2026-07-01T21:07:55Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:07:55Z" name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs.18be468d53ed438c namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: ce093ef9-cc0a-4818-bd9f-5c3d2df28cf0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: d3ebb858-b3ad-44ca-9ee6-ef6508714873 kind: Event lastTimestamp: "2026-07-01T21:07:55Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cc7b8fac71b36ebfd1661675b86a84ade93db501973e87758605bd27f47937cb" already present on machine metadata: creationTimestamp: "2026-07-01T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:55Z" name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs.18be468d555680e7 namespace: kserve-ci-e2e-test resourceVersion: "20700" uid: db08ae72-dc1c-4389-8ef3-bd98e0ada6b7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: d3ebb858-b3ad-44ca-9ee6-ef6508714873 kind: Event lastTimestamp: "2026-07-01T21:07:55Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:55Z" name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs.18be468d57f91afa namespace: kserve-ci-e2e-test resourceVersion: "20701" uid: c0f1417c-1c98-4bcf-8d79-76a489f557eb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: d3ebb858-b3ad-44ca-9ee6-ef6508714873 kind: Event lastTimestamp: "2026-07-01T21:07:55Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T21:07:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:55Z" name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs.18be468d586ac058 namespace: kserve-ci-e2e-test resourceVersion: "20702" uid: a75f6fd8-874e-4c54-99c4-e85d27daa74f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: d3ebb858-b3ad-44ca-9ee6-ef6508714873 kind: Event lastTimestamp: "2026-07-01T21:07:59Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-01T21:07:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:59Z" name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs.18be468e5a6f843c namespace: kserve-ci-e2e-test resourceVersion: "20737" uid: 29a05e88-de85-4f61-baee-6dcd079b7900 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: d3ebb858-b3ad-44ca-9ee6-ef6508714873 kind: Event lastTimestamp: "2026-07-01T21:07:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T21:07:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:59Z" name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs.18be468e5daed461 namespace: kserve-ci-e2e-test resourceVersion: "20739" uid: 61a42b8f-0666-49e3-ae8f-3e7640c3d9c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: d3ebb858-b3ad-44ca-9ee6-ef6508714873 kind: Event lastTimestamp: "2026-07-01T21:07:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T21:07:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:59Z" name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs.18be468e5e21895b namespace: kserve-ci-e2e-test resourceVersion: "20740" uid: a95cb8b4-e823-4d9a-b8e9-225653829cd5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: d3ebb858-b3ad-44ca-9ee6-ef6508714873 kind: Event lastTimestamp: "2026-07-01T21:07:59Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T21:07:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:59Z" name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs.18be468e5e2aba02 namespace: kserve-ci-e2e-test resourceVersion: "20741" uid: fed97a9a-51ea-48d7-9190-f3b79384fe77 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: d3ebb858-b3ad-44ca-9ee6-ef6508714873 kind: Event lastTimestamp: "2026-07-01T21:07:59Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T21:07:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:59Z" name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs.18be468e60d2f575 namespace: kserve-ci-e2e-test resourceVersion: "20742" uid: ee337509-6448-40c3-b918-00ebeb92ba1c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: d3ebb858-b3ad-44ca-9ee6-ef6508714873 kind: Event lastTimestamp: "2026-07-01T21:07:59Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:07:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:07:59Z" name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs.18be468e6140f3c4 namespace: kserve-ci-e2e-test resourceVersion: "20743" uid: ae5b21da-35f0-486b-a5d5-0fddfc0ab462 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T21:08:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: d3ebb858-b3ad-44ca-9ee6-ef6508714873 kind: Event lastTimestamp: "2026-07-01T21:09:17Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:08:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:09:17Z" name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs.18be468ed249b2b9 namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: 2ae8c730-1044-4bf8-a6ff-2f107f030506 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:09:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: d3ebb858-b3ad-44ca-9ee6-ef6508714873 kind: Event lastTimestamp: "2026-07-01T21:09:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T21:09:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:09:14Z" name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs.18be469fb4c17a9f namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 747884ec-f1da-4b81-a3ba-b0a6eedc7101 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:09:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: d3ebb858-b3ad-44ca-9ee6-ef6508714873 kind: Event lastTimestamp: "2026-07-01T21:09:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:09:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:09:14Z" name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs.18be469fb4c20658 namespace: kserve-ci-e2e-test resourceVersion: "21300" uid: 8f3d6b57-3102-4a48-97d2-48640924bac3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:09:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: d3ebb858-b3ad-44ca-9ee6-ef6508714873 kind: Event lastTimestamp: "2026-07-01T21:09:17Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:09:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:09:17Z" name: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs.18be46a084689736 namespace: kserve-ci-e2e-test resourceVersion: "21325" uid: a5e07121-91e7-4a29-b569-bb2ef3284752 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-92cf8-predictor-d4757fc6b namespace: kserve-ci-e2e-test resourceVersion: "20665" uid: 2f74a908-5c67-4f84-b7d8-c3739c073a02 kind: Event lastTimestamp: "2026-07-01T21:07:54Z" message: 'Created pod: raw-sklearn-92cf8-predictor-d4757fc6b-s49vs' metadata: creationTimestamp: "2026-07-01T21:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:54Z" name: raw-sklearn-92cf8-predictor-d4757fc6b.18be468d165e01d3 namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: 6815b165-5575-4b6e-8ce6-f459e2904cc5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-92cf8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: 288c66aa-cad3-4d59-941a-a16ec52a530b kind: Event lastTimestamp: "2026-07-01T21:07:54Z" message: Scaled up replica set raw-sklearn-92cf8-predictor-d4757fc6b from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:07:54Z" name: raw-sklearn-92cf8-predictor.18be468d0f9e68e2 namespace: kserve-ci-e2e-test resourceVersion: "20667" uid: a1dab947-cea6-4816-a034-4e600a9ae25e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:08:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-92cf8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20663" uid: 14df4cb1-6607-4d21-9c1a-11f1d8028f71 kind: Event lastTimestamp: "2026-07-01T21:08:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:08:39Z" name: raw-sklearn-92cf8-predictor.18be46908e8f98bd namespace: kserve-ci-e2e-test resourceVersion: "21022" uid: 9fc01305-1282-4fe5-b611-52ee6a2e1aa2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T21:08:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-92cf8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20663" uid: 14df4cb1-6607-4d21-9c1a-11f1d8028f71 kind: Event lastTimestamp: "2026-07-01T21:08:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:08:39Z" name: raw-sklearn-92cf8-predictor.18be46908efa76a3 namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: a7046c7d-7ffc-4993-bbcb-72876d219d61 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:08:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-92cf8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: 14df4cb1-6607-4d21-9c1a-11f1d8028f71 kind: Event lastTimestamp: "2026-07-01T21:09:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:09:09Z" name: raw-sklearn-92cf8-predictor.18be469b0c631a7e namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 51cf07d8-b52a-4a4c-9e89-79d4f81a044a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:08:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-92cf8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: 14df4cb1-6607-4d21-9c1a-11f1d8028f71 kind: Event lastTimestamp: "2026-07-01T21:09:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:08:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:09:09Z" name: raw-sklearn-92cf8-predictor.18be469b0ccdde10 namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: d76bd1c6-f775-4ec7-9645-d73561905890 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T21:07:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-92cf8 namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 586ca410-78c5-4f2b-b1a0-243c02c1aa10 kind: Event lastTimestamp: "2026-07-01T21:09:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T21:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:09:07Z" name: raw-sklearn-92cf8.18be468d0f9b6d88 namespace: kserve-ci-e2e-test resourceVersion: "21234" uid: c0689da9-1167-4b43-b2ad-658824ebf6cd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:07:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-92cf8 namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 586ca410-78c5-4f2b-b1a0-243c02c1aa10 kind: Event lastTimestamp: "2026-07-01T21:07:54Z" message: 'Failed to update status for InferenceService "raw-sklearn-92cf8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-92cf8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:07:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:07:54Z" name: raw-sklearn-92cf8.18be468d104bcc3e namespace: kserve-ci-e2e-test resourceVersion: "20670" uid: 0e6319e0-1b27-4f38-970a-f3d8198522da reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:09:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-92cf8 namespace: kserve-ci-e2e-test resourceVersion: "21232" uid: 586ca410-78c5-4f2b-b1a0-243c02c1aa10 kind: Event lastTimestamp: "2026-07-01T21:09:07Z" message: InferenceService [raw-sklearn-92cf8] is Ready metadata: creationTimestamp: "2026-07-01T21:09:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:09:07Z" name: raw-sklearn-92cf8.18be469e374fed0b namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: 41e266ec-4d00-4a87-84aa-a81a6fbcd741 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T21:09:19.403720Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc namespace: kserve-ci-e2e-test resourceVersion: "21362" uid: 28adbd4b-9edd-49bf-aede-1ad7f0e0366f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc to ip-10-0-135-188.ec2.internal metadata: creationTimestamp: "2026-07-01T21:09:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T21:09:19Z" name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc.18be46a0e9099714 namespace: kserve-ci-e2e-test resourceVersion: "21368" uid: 3d77fa66-10bb-4f8c-b008-998996dc5507 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-58dc7469c8-ljvgg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:09:19Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc namespace: kserve-ci-e2e-test resourceVersion: "21372" uid: 28adbd4b-9edd-49bf-aede-1ad7f0e0366f kind: Event lastTimestamp: "2026-07-01T21:09:19Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T21:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T21:09:19Z" name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc.18be46a1026114eb namespace: kserve-ci-e2e-test resourceVersion: "21380" uid: 1fef5bdd-7db7-45bc-8010-7d66f0705b23 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:09:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc namespace: kserve-ci-e2e-test resourceVersion: "21364" uid: 28adbd4b-9edd-49bf-aede-1ad7f0e0366f kind: Event lastTimestamp: "2026-07-01T21:09:19Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cc7b8fac71b36ebfd1661675b86a84ade93db501973e87758605bd27f47937cb" already present on machine metadata: creationTimestamp: "2026-07-01T21:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:09:19Z" name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc.18be46a103d82f04 namespace: kserve-ci-e2e-test resourceVersion: "21382" uid: dbe23fb0-f4cf-485c-a1e0-1597297eb6e8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:09:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc namespace: kserve-ci-e2e-test resourceVersion: "21364" uid: 28adbd4b-9edd-49bf-aede-1ad7f0e0366f kind: Event lastTimestamp: "2026-07-01T21:09:19Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T21:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:09:19Z" name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc.18be46a106df6590 namespace: kserve-ci-e2e-test resourceVersion: "21383" uid: 11911bc3-226b-4378-8ff6-018628a7b440 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:09:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc namespace: kserve-ci-e2e-test resourceVersion: "21364" uid: 28adbd4b-9edd-49bf-aede-1ad7f0e0366f kind: Event lastTimestamp: "2026-07-01T21:09:19Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T21:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:09:19Z" name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc.18be46a107540cc8 namespace: kserve-ci-e2e-test resourceVersion: "21384" uid: 607fc930-03f8-4ae1-8636-38710dbb6e25 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:09:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc namespace: kserve-ci-e2e-test resourceVersion: "21364" uid: 28adbd4b-9edd-49bf-aede-1ad7f0e0366f kind: Event lastTimestamp: "2026-07-01T21:09:24Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1683" already present on machine metadata: creationTimestamp: "2026-07-01T21:09:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:09:24Z" name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc.18be46a1fcc5c82a namespace: kserve-ci-e2e-test resourceVersion: "21418" uid: 7c014d4a-785c-47f9-a767-96c42694e268 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:09:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc namespace: kserve-ci-e2e-test resourceVersion: "21364" uid: 28adbd4b-9edd-49bf-aede-1ad7f0e0366f kind: Event lastTimestamp: "2026-07-01T21:09:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T21:09:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:09:24Z" name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc.18be46a1ffd85558 namespace: kserve-ci-e2e-test resourceVersion: "21420" uid: 5c9e459d-9e6b-4307-9f61-f6f72d6c3da2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:09:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc namespace: kserve-ci-e2e-test resourceVersion: "21364" uid: 28adbd4b-9edd-49bf-aede-1ad7f0e0366f kind: Event lastTimestamp: "2026-07-01T21:09:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T21:09:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:09:24Z" name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc.18be46a2004f7d36 namespace: kserve-ci-e2e-test resourceVersion: "21423" uid: da0a3f8f-eb01-4c61-98d0-d5bb788c0330 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:09:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc namespace: kserve-ci-e2e-test resourceVersion: "21364" uid: 28adbd4b-9edd-49bf-aede-1ad7f0e0366f kind: Event lastTimestamp: "2026-07-01T21:09:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T21:09:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:09:24Z" name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc.18be46a20058d13e namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: 18fedcbc-6956-4f07-9ed3-ef536a8b54b8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:09:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc namespace: kserve-ci-e2e-test resourceVersion: "21364" uid: 28adbd4b-9edd-49bf-aede-1ad7f0e0366f kind: Event lastTimestamp: "2026-07-01T21:09:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T21:09:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:09:24Z" name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc.18be46a202f96db5 namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 9d2a2a0f-88ce-48be-a116-b8b9e41cc5b9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:09:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc namespace: kserve-ci-e2e-test resourceVersion: "21364" uid: 28adbd4b-9edd-49bf-aede-1ad7f0e0366f kind: Event lastTimestamp: "2026-07-01T21:09:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:09:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:09:24Z" name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc.18be46a2036edf2d namespace: kserve-ci-e2e-test resourceVersion: "21427" uid: ea1fe11a-8cee-4e56-b0c5-e9a37b400626 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T21:09:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc namespace: kserve-ci-e2e-test resourceVersion: "21364" uid: 28adbd4b-9edd-49bf-aede-1ad7f0e0366f kind: Event lastTimestamp: "2026-07-01T21:10:51Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:09:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:10:51Z" name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc.18be46a238cf3e57 namespace: kserve-ci-e2e-test resourceVersion: "22064" uid: 45156d90-04d9-49a3-87c1-3b2c5489dd8c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:10:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc namespace: kserve-ci-e2e-test resourceVersion: "21364" uid: 28adbd4b-9edd-49bf-aede-1ad7f0e0366f kind: Event lastTimestamp: "2026-07-01T21:10:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T21:10:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:10:49Z" name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc.18be46b5e61fe11b namespace: kserve-ci-e2e-test resourceVersion: "22052" uid: 819b5ae9-a8fd-4e57-845a-bb3975b312f7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:10:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc namespace: kserve-ci-e2e-test resourceVersion: "21364" uid: 28adbd4b-9edd-49bf-aede-1ad7f0e0366f kind: Event lastTimestamp: "2026-07-01T21:10:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T21:10:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:10:49Z" name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc.18be46b5e6203fd0 namespace: kserve-ci-e2e-test resourceVersion: "22053" uid: fd6ccf01-3ea8-48d2-9207-7a998e2c47ff reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:10:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc namespace: kserve-ci-e2e-test resourceVersion: "21364" uid: 28adbd4b-9edd-49bf-aede-1ad7f0e0366f kind: Event lastTimestamp: "2026-07-01T21:10:51Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T21:10:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T21:10:51Z" name: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc.18be46b63efd8e68 namespace: kserve-ci-e2e-test resourceVersion: "22063" uid: 73fd1bd8-1efa-4144-ad6c-7048903da603 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-188.ec2.internal source: component: kubelet host: ip-10-0-135-188.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:09:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-830cb-predictor-56b49bc498 namespace: kserve-ci-e2e-test resourceVersion: "21353" uid: fc095d46-08e0-4ab4-9c32-3b16c7adc73b kind: Event lastTimestamp: "2026-07-01T21:09:19Z" message: 'Created pod: raw-sklearn-runtime-830cb-predictor-56b49bc498-h2rwc' metadata: creationTimestamp: "2026-07-01T21:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:09:19Z" name: raw-sklearn-runtime-830cb-predictor-56b49bc498.18be46a0e8686ab7 namespace: kserve-ci-e2e-test resourceVersion: "21363" uid: 1728d153-cfe6-4a74-9b89-374a3d56bf09 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:09:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-830cb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21351" uid: e4e0258c-28cc-4099-9add-a24a560403cb kind: Event lastTimestamp: "2026-07-01T21:09:19Z" message: Scaled up replica set raw-sklearn-runtime-830cb-predictor-56b49bc498 from 0 to 1 metadata: creationTimestamp: "2026-07-01T21:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:09:19Z" name: raw-sklearn-runtime-830cb-predictor.18be46a0e4ad233e namespace: kserve-ci-e2e-test resourceVersion: "21354" uid: dd9c34f0-3686-48db-8a81-e5357a087554 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:09:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-830cb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: db4fa3c7-fdf0-42a7-9e8b-932e324d033b kind: Event lastTimestamp: "2026-07-01T21:09:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:09:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:09:49Z" name: raw-sklearn-runtime-830cb-predictor.18be46a463d6526f namespace: kserve-ci-e2e-test resourceVersion: "21609" uid: 41e47adb-ab82-46a9-a935-cb6ff692bdea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T21:09:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-830cb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: db4fa3c7-fdf0-42a7-9e8b-932e324d033b kind: Event lastTimestamp: "2026-07-01T21:09:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T21:09:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:09:49Z" name: raw-sklearn-runtime-830cb-predictor.18be46a4643d8a53 namespace: kserve-ci-e2e-test resourceVersion: "21610" uid: 7ef20edf-996e-4cd0-9b98-1ea4e1dd2594 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T21:10:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-830cb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21501" uid: db4fa3c7-fdf0-42a7-9e8b-932e324d033b kind: Event lastTimestamp: "2026-07-01T21:10:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:10:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:10:49Z" name: raw-sklearn-runtime-830cb-predictor.18be46ab62ce6f6e namespace: kserve-ci-e2e-test resourceVersion: "22035" uid: e223b157-6bf1-42c2-a24a-fea307673831 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T21:10:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-830cb-predictor namespace: kserve-ci-e2e-test resourceVersion: "21501" uid: db4fa3c7-fdf0-42a7-9e8b-932e324d033b kind: Event lastTimestamp: "2026-07-01T21:10:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T21:10:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T21:10:49Z" name: raw-sklearn-runtime-830cb-predictor.18be46ab6317a276 namespace: kserve-ci-e2e-test resourceVersion: "22036" uid: 38e2fec8-4c88-4457-bd69-c28ed4f6628b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T21:09:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-830cb namespace: kserve-ci-e2e-test resourceVersion: "21344" uid: 81864e01-ed99-47a7-8783-c0fcaa32c5a7 kind: Event lastTimestamp: "2026-07-01T21:10:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T21:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:10:41Z" name: raw-sklearn-runtime-830cb.18be46a0e4b0935d namespace: kserve-ci-e2e-test resourceVersion: "21978" uid: 45a33475-4ca9-4ba9-86be-307e3894948e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:09:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-830cb namespace: kserve-ci-e2e-test resourceVersion: "21344" uid: 81864e01-ed99-47a7-8783-c0fcaa32c5a7 kind: Event lastTimestamp: "2026-07-01T21:09:19Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-830cb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-830cb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T21:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:09:19Z" name: raw-sklearn-runtime-830cb.18be46a0e520d7a2 namespace: kserve-ci-e2e-test resourceVersion: "21358" uid: 05140e51-23dc-47ed-bab8-0c812c8a874c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T21:10:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-830cb namespace: kserve-ci-e2e-test resourceVersion: "21974" uid: 81864e01-ed99-47a7-8783-c0fcaa32c5a7 kind: Event lastTimestamp: "2026-07-01T21:10:41Z" message: InferenceService [raw-sklearn-runtime-830cb] is Ready metadata: creationTimestamp: "2026-07-01T21:10:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T21:10:41Z" name: raw-sklearn-runtime-830cb.18be46b3f163a5b5 namespace: kserve-ci-e2e-test resourceVersion: "21976" uid: 42aa852b-d991-45b7-bd7c-7e4cf0dee80e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "22295"