--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:56:37.872806Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-1558a9-predictor-7d45464cf5-9p4c2 namespace: kserve-ci-e2e-test resourceVersion: "20527" uid: 0d1d570e-cd70-45f3-b7ea-1ff67e824ce0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-1558a9-predictor-7d45464cf5-9p4c2 to ip-10-0-139-250.ec2.internal metadata: creationTimestamp: "2026-07-06T20:56:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:56:37Z" name: isvc-init-fail-1558a9-predictor-7d45464cf5-9p4c2.18bfced670cd0ce8 namespace: kserve-ci-e2e-test resourceVersion: "20533" uid: a5780f4c-3176-46a9-b946-9e0c71466cad reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-95cfcc986-db5pj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:38Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-1558a9-predictor-7d45464cf5-9p4c2 namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: 0d1d570e-cd70-45f3-b7ea-1ff67e824ce0 kind: Event lastTimestamp: "2026-07-06T20:56:38Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:56:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:56:38Z" name: isvc-init-fail-1558a9-predictor-7d45464cf5-9p4c2.18bfced68a73912b namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: 0f8c25ed-8c39-4419-941c-bc8d6ba0a853 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:56:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-1558a9-predictor-7d45464cf5-9p4c2 namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: 0d1d570e-cd70-45f3-b7ea-1ff67e824ce0 kind: Event lastTimestamp: "2026-07-06T20:56:45Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:9b7f9014cc3727e4fd95a945572d3f70017252d596bd8941a1887e4a1514875c" already present on machine metadata: creationTimestamp: "2026-07-06T20:56:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:45Z" name: isvc-init-fail-1558a9-predictor-7d45464cf5-9p4c2.18bfced68c046c09 namespace: kserve-ci-e2e-test resourceVersion: "20605" uid: ad672b0a-f4c3-4941-b93d-fc20725e36d2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:56:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-1558a9-predictor-7d45464cf5-9p4c2 namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: 0d1d570e-cd70-45f3-b7ea-1ff67e824ce0 kind: Event lastTimestamp: "2026-07-06T20:56:45Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-06T20:56:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:45Z" name: isvc-init-fail-1558a9-predictor-7d45464cf5-9p4c2.18bfced68ea37f82 namespace: kserve-ci-e2e-test resourceVersion: "20607" uid: 09dc45d1-e446-49f8-9b96-1cf10356ec72 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:56:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-1558a9-predictor-7d45464cf5-9p4c2 namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: 0d1d570e-cd70-45f3-b7ea-1ff67e824ce0 kind: Event lastTimestamp: "2026-07-06T20:56:45Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-06T20:56:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:45Z" name: isvc-init-fail-1558a9-predictor-7d45464cf5-9p4c2.18bfced68f1d7799 namespace: kserve-ci-e2e-test resourceVersion: "20609" uid: 70d0666a-77ca-492e-ad16-ed667efc0650 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-1558a9-predictor-7d45464cf5-9p4c2 namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: 0d1d570e-cd70-45f3-b7ea-1ff67e824ce0 kind: Event lastTimestamp: "2026-07-06T20:56:47Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-07-06T20:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:47Z" name: isvc-init-fail-1558a9-predictor-7d45464cf5-9p4c2.18bfced8c4005696 namespace: kserve-ci-e2e-test resourceVersion: "20644" uid: 127a3410-96b4-4d84-a5c4-89296b4d76fc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-1558a9-predictor-7d45464cf5 namespace: kserve-ci-e2e-test resourceVersion: "20519" uid: 6b0aae96-f96c-4ed2-8a7e-df2924fd4015 kind: Event lastTimestamp: "2026-07-06T20:56:37Z" message: 'Created pod: isvc-init-fail-1558a9-predictor-7d45464cf5-9p4c2' metadata: creationTimestamp: "2026-07-06T20:56:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:56:37Z" name: isvc-init-fail-1558a9-predictor-7d45464cf5.18bfced66fda08f2 namespace: kserve-ci-e2e-test resourceVersion: "20528" uid: f92e1e9d-8baa-47b5-9932-ca7c108bb341 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-1558a9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20517" uid: c5a4067d-8858-46ef-ad31-0add98da5158 kind: Event lastTimestamp: "2026-07-06T20:56:37Z" message: Scaled up replica set isvc-init-fail-1558a9-predictor-7d45464cf5 from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:56:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:56:37Z" name: isvc-init-fail-1558a9-predictor.18bfced66c104ed9 namespace: kserve-ci-e2e-test resourceVersion: "20521" uid: 34a59a30-67b7-4041-82bf-31262aba31e8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-06T20:56:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-1558a9 namespace: kserve-ci-e2e-test resourceVersion: "20510" uid: f9d1f7d6-624e-42f4-8fc8-26648d0426d3 kind: Event lastTimestamp: "2026-07-06T20:56:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:56:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:56:45Z" name: isvc-init-fail-1558a9.18bfced66c2e2a9d namespace: kserve-ci-e2e-test resourceVersion: "20612" uid: ab50b1df-c592-4b80-b5f7-653e73476473 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-1558a9 namespace: kserve-ci-e2e-test resourceVersion: "20510" uid: f9d1f7d6-624e-42f4-8fc8-26648d0426d3 kind: Event lastTimestamp: "2026-07-06T20:56:37Z" message: 'Failed to update status for InferenceService "isvc-init-fail-1558a9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-1558a9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:56:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:56:37Z" name: isvc-init-fail-1558a9.18bfced66cec163e namespace: kserve-ci-e2e-test resourceVersion: "20524" uid: eaccf22e-320a-4f61-ba81-35677c26a884 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-1558a9 namespace: kserve-ci-e2e-test resourceVersion: "20510" uid: f9d1f7d6-624e-42f4-8fc8-26648d0426d3 kind: Event lastTimestamp: "2026-07-06T20:56:37Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-1558a9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:56:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:56:37Z" name: isvc-init-fail-1558a9.18bfced66cec6975 namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: 9aef0542-2101-4572-a139-28b465a3c164 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:50:17.821961Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw namespace: kserve-ci-e2e-test resourceVersion: "17443" uid: a5ba0a95-139b-4d29-9b87-2edebbcae931 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw to ip-10-0-139-250.ec2.internal metadata: creationTimestamp: "2026-07-06T20:50:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:50:17Z" name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw.18bfce7df4016bb5 namespace: kserve-ci-e2e-test resourceVersion: "17447" uid: 54f00f27-4c86-494c-8c5d-801b592fa969 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-95cfcc986-db5pj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:18Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw namespace: kserve-ci-e2e-test resourceVersion: "17455" uid: a5ba0a95-139b-4d29-9b87-2edebbcae931 kind: Event lastTimestamp: "2026-07-06T20:50:18Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:50:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:50:18Z" name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw.18bfce7e0da7c579 namespace: kserve-ci-e2e-test resourceVersion: "17463" uid: c7a66929-4ad0-47c7-944e-6693eb63910c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw namespace: kserve-ci-e2e-test resourceVersion: "17445" uid: a5ba0a95-139b-4d29-9b87-2edebbcae931 kind: Event lastTimestamp: "2026-07-06T20:50:18Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:9b7f9014cc3727e4fd95a945572d3f70017252d596bd8941a1887e4a1514875c" already present on machine metadata: creationTimestamp: "2026-07-06T20:50:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:18Z" name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw.18bfce7e0f8ef133 namespace: kserve-ci-e2e-test resourceVersion: "17465" uid: 743065d9-4a4f-4663-9b81-c04af8c2405f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw namespace: kserve-ci-e2e-test resourceVersion: "17445" uid: a5ba0a95-139b-4d29-9b87-2edebbcae931 kind: Event lastTimestamp: "2026-07-06T20:50:18Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-06T20:50:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:18Z" name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw.18bfce7e12b3bbb5 namespace: kserve-ci-e2e-test resourceVersion: "17467" uid: e469e912-0782-4841-8747-f1e529635571 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw namespace: kserve-ci-e2e-test resourceVersion: "17445" uid: a5ba0a95-139b-4d29-9b87-2edebbcae931 kind: Event lastTimestamp: "2026-07-06T20:50:18Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-06T20:50:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:18Z" name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw.18bfce7e13280995 namespace: kserve-ci-e2e-test resourceVersion: "17468" uid: 9caf4f08-a802-4233-a54e-56ed0e13ac70 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw namespace: kserve-ci-e2e-test resourceVersion: "17445" uid: a5ba0a95-139b-4d29-9b87-2edebbcae931 kind: Event lastTimestamp: "2026-07-06T20:50:22Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1703" already present on machine metadata: creationTimestamp: "2026-07-06T20:50:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:22Z" name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw.18bfce7eff00e49c namespace: kserve-ci-e2e-test resourceVersion: "17508" uid: 5ecf0cab-ce66-4a9f-a6e6-b39bb129253b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw namespace: kserve-ci-e2e-test resourceVersion: "17445" uid: a5ba0a95-139b-4d29-9b87-2edebbcae931 kind: Event lastTimestamp: "2026-07-06T20:50:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:50:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:22Z" name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw.18bfce7f023e5ede namespace: kserve-ci-e2e-test resourceVersion: "17511" uid: b063a529-000c-4a47-b711-ffe04e61ff90 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw namespace: kserve-ci-e2e-test resourceVersion: "17445" uid: a5ba0a95-139b-4d29-9b87-2edebbcae931 kind: Event lastTimestamp: "2026-07-06T20:50:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:50:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:22Z" name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw.18bfce7f02bc5e7f namespace: kserve-ci-e2e-test resourceVersion: "17512" uid: 3ce5361b-0fc1-48bd-a02b-de4c74e156a1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw namespace: kserve-ci-e2e-test resourceVersion: "17445" uid: a5ba0a95-139b-4d29-9b87-2edebbcae931 kind: Event lastTimestamp: "2026-07-06T20:50:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T20:50:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:22Z" name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw.18bfce7f02c69477 namespace: kserve-ci-e2e-test resourceVersion: "17513" uid: 92066471-1357-4d39-af53-1f14bda6d092 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw namespace: kserve-ci-e2e-test resourceVersion: "17445" uid: a5ba0a95-139b-4d29-9b87-2edebbcae931 kind: Event lastTimestamp: "2026-07-06T20:50:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:50:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:22Z" name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw.18bfce7f05a455ff namespace: kserve-ci-e2e-test resourceVersion: "17516" uid: df778eca-e13b-43a7-abad-0228ff40f586 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw namespace: kserve-ci-e2e-test resourceVersion: "17445" uid: a5ba0a95-139b-4d29-9b87-2edebbcae931 kind: Event lastTimestamp: "2026-07-06T20:50:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:50:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:22Z" name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw.18bfce7f061aeec5 namespace: kserve-ci-e2e-test resourceVersion: "17518" uid: 621bd57b-c9fa-4798-80ad-c4524fc0149b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw namespace: kserve-ci-e2e-test resourceVersion: "17445" uid: a5ba0a95-139b-4d29-9b87-2edebbcae931 kind: Event lastTimestamp: "2026-07-06T20:50:22Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:0706c86054be2f22ee66c8739e97daebd0094b98a978f91cd3fb4f59862bad3f" already present on machine metadata: creationTimestamp: "2026-07-06T20:50:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:22Z" name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw.18bfce7f0624af29 namespace: kserve-ci-e2e-test resourceVersion: "17519" uid: fc9f7069-e383-4574-8f6c-dd1f35416ef2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw namespace: kserve-ci-e2e-test resourceVersion: "17445" uid: a5ba0a95-139b-4d29-9b87-2edebbcae931 kind: Event lastTimestamp: "2026-07-06T20:50:22Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-06T20:50:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:22Z" name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw.18bfce7f0988e7c3 namespace: kserve-ci-e2e-test resourceVersion: "17520" uid: 9e6507c6-2d7f-428c-a776-184f9b1c54f6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw namespace: kserve-ci-e2e-test resourceVersion: "17445" uid: a5ba0a95-139b-4d29-9b87-2edebbcae931 kind: Event lastTimestamp: "2026-07-06T20:50:22Z" message: Started container agent metadata: creationTimestamp: "2026-07-06T20:50:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:22Z" name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw.18bfce7f0a00d5d7 namespace: kserve-ci-e2e-test resourceVersion: "17521" uid: 28a24743-d2ac-4b57-9e10-d840bbf5e751 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-06T20:50:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw namespace: kserve-ci-e2e-test resourceVersion: "17445" uid: a5ba0a95-139b-4d29-9b87-2edebbcae931 kind: Event lastTimestamp: "2026-07-06T20:51:59Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:50:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:51:59Z" name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw.18bfce7f3b0ad586 namespace: kserve-ci-e2e-test resourceVersion: "18324" uid: a54bf055-5c29-47bc-bbb3-c52ebeb06c09 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-06T20:50:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw namespace: kserve-ci-e2e-test resourceVersion: "17445" uid: a5ba0a95-139b-4d29-9b87-2edebbcae931 kind: Event lastTimestamp: "2026-07-06T20:52:09Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-06T20:50:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:52:09Z" name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw.18bfce7f3b15847b namespace: kserve-ci-e2e-test resourceVersion: "18419" uid: e0841285-68dc-4ed7-bcf4-e39593cfb282 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:51:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw namespace: kserve-ci-e2e-test resourceVersion: "17445" uid: a5ba0a95-139b-4d29-9b87-2edebbcae931 kind: Event lastTimestamp: "2026-07-06T20:51:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T20:51:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:51:53Z" name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw.18bfce942545c088 namespace: kserve-ci-e2e-test resourceVersion: "18232" uid: 300840b0-962f-46c3-91e4-2479e9c960e8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:51:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw namespace: kserve-ci-e2e-test resourceVersion: "17445" uid: a5ba0a95-139b-4d29-9b87-2edebbcae931 kind: Event lastTimestamp: "2026-07-06T20:51:53Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-06T20:51:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:51:53Z" name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw.18bfce9425460259 namespace: kserve-ci-e2e-test resourceVersion: "18234" uid: 61d55d43-80e0-4677-b30e-3a5b8b340c18 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:51:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw namespace: kserve-ci-e2e-test resourceVersion: "17445" uid: a5ba0a95-139b-4d29-9b87-2edebbcae931 kind: Event lastTimestamp: "2026-07-06T20:51:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:51:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:51:53Z" name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw.18bfce9425468cdb namespace: kserve-ci-e2e-test resourceVersion: "18240" uid: ac834f3c-a7e2-41b7-9f4b-466aebc5a9df reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T20:51:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw namespace: kserve-ci-e2e-test resourceVersion: "17445" uid: a5ba0a95-139b-4d29-9b87-2edebbcae931 kind: Event lastTimestamp: "2026-07-06T20:52:09Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:51:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:52:09Z" name: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw.18bfce946b3c44e6 namespace: kserve-ci-e2e-test resourceVersion: "18418" uid: 8d14ef81-c5f1-4b29-a94a-13fedbefa6d1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-e7cb2-predictor-6748c49bb5 namespace: kserve-ci-e2e-test resourceVersion: "17432" uid: 35727f26-540e-423a-996f-62197397c019 kind: Event lastTimestamp: "2026-07-06T20:50:17Z" message: 'Created pod: isvc-logger-raw-e7cb2-predictor-6748c49bb5-p7bgw' metadata: creationTimestamp: "2026-07-06T20:50:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:50:17Z" name: isvc-logger-raw-e7cb2-predictor-6748c49bb5.18bfce7df306e0da namespace: kserve-ci-e2e-test resourceVersion: "17444" uid: 720b7bff-4bf3-47d1-8d6a-58ad2a594f85 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-e7cb2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17430" uid: 4e5a2639-e2b9-4780-bc56-226a1079426d kind: Event lastTimestamp: "2026-07-06T20:50:17Z" message: Scaled up replica set isvc-logger-raw-e7cb2-predictor-6748c49bb5 from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:50:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:50:17Z" name: isvc-logger-raw-e7cb2-predictor.18bfce7dec5968a9 namespace: kserve-ci-e2e-test resourceVersion: "17435" uid: bf0913a4-9c61-4e73-9cf0-8adc20128587 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T20:50:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-e7cb2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17431" uid: aa823399-6659-4b7f-8717-cef6d122b1b3 kind: Event lastTimestamp: "2026-07-06T20:51:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:50:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:51:17Z" name: isvc-logger-raw-e7cb2-predictor.18bfce816bf4ae4f namespace: kserve-ci-e2e-test resourceVersion: "17934" uid: 87588f5c-8baa-4102-a618-d43b7990bb05 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T20:50:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-e7cb2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17431" uid: aa823399-6659-4b7f-8717-cef6d122b1b3 kind: Event lastTimestamp: "2026-07-06T20:51:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:50:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:51:17Z" name: isvc-logger-raw-e7cb2-predictor.18bfce816c9b6956 namespace: kserve-ci-e2e-test resourceVersion: "17935" uid: 11e2d59d-0ef2-48f4-9124-d45313a75290 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:51:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-e7cb2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17609" uid: aa823399-6659-4b7f-8717-cef6d122b1b3 kind: Event lastTimestamp: "2026-07-06T20:51:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:51:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:51:47Z" name: isvc-logger-raw-e7cb2-predictor.18bfce8f69299673 namespace: kserve-ci-e2e-test resourceVersion: "18154" uid: 56525d29-3a11-4189-b84a-16de0b740b8f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:51:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-e7cb2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17609" uid: aa823399-6659-4b7f-8717-cef6d122b1b3 kind: Event lastTimestamp: "2026-07-06T20:51:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:51:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:51:47Z" name: isvc-logger-raw-e7cb2-predictor.18bfce8f69f5be00 namespace: kserve-ci-e2e-test resourceVersion: "18155" uid: 15fcaed4-f031-402c-89cc-6bf1c429223e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-06T20:50:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-e7cb2 namespace: kserve-ci-e2e-test resourceVersion: "17420" uid: 9700fbba-c0ff-4e57-83a6-8d7f788be0e7 kind: Event lastTimestamp: "2026-07-06T20:51:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:50:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:51:39Z" name: isvc-logger-raw-e7cb2.18bfce7dec5c6404 namespace: kserve-ci-e2e-test resourceVersion: "18092" uid: f323b0c4-e7f3-4fc3-98e7-db2d7126e936 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-e7cb2 namespace: kserve-ci-e2e-test resourceVersion: "17420" uid: 9700fbba-c0ff-4e57-83a6-8d7f788be0e7 kind: Event lastTimestamp: "2026-07-06T20:50:17Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-e7cb2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-e7cb2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:50:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:50:17Z" name: isvc-logger-raw-e7cb2.18bfce7ded67917d namespace: kserve-ci-e2e-test resourceVersion: "17437" uid: aca08dc7-fe74-491d-abc5-ad10708137d8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:51:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-e7cb2 namespace: kserve-ci-e2e-test resourceVersion: "18089" uid: 9700fbba-c0ff-4e57-83a6-8d7f788be0e7 kind: Event lastTimestamp: "2026-07-06T20:51:39Z" message: InferenceService [isvc-logger-raw-e7cb2] is Ready metadata: creationTimestamp: "2026-07-06T20:51:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:51:39Z" name: isvc-logger-raw-e7cb2.18bfce90f6ad1fb4 namespace: kserve-ci-e2e-test resourceVersion: "18090" uid: b46753a3-62b5-4dda-b909-8f295b915ebc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:54:48.413077Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-f8d41a-predictor-656db64bb8-495pz namespace: kserve-ci-e2e-test resourceVersion: "19583" uid: 2badce2e-d8c6-408c-8b4a-e55b3f7d3dc1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-f8d41a-predictor-656db64bb8-495pz to ip-10-0-139-250.ec2.internal metadata: creationTimestamp: "2026-07-06T20:54:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:54:48Z" name: isvc-primary-f8d41a-predictor-656db64bb8-495pz.18bfcebcf47e109d namespace: kserve-ci-e2e-test resourceVersion: "19588" uid: 906d8a6b-5881-4de8-b86a-9430cbd04b2a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-95cfcc986-db5pj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:54:48Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-f8d41a-predictor-656db64bb8-495pz namespace: kserve-ci-e2e-test resourceVersion: "19591" uid: 2badce2e-d8c6-408c-8b4a-e55b3f7d3dc1 kind: Event lastTimestamp: "2026-07-06T20:54:48Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:54:48Z" name: isvc-primary-f8d41a-predictor-656db64bb8-495pz.18bfcebd0da23d53 namespace: kserve-ci-e2e-test resourceVersion: "19599" uid: cfbda2ef-d0c3-4e39-80ce-485568246e1c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:54:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-f8d41a-predictor-656db64bb8-495pz namespace: kserve-ci-e2e-test resourceVersion: "19584" uid: 2badce2e-d8c6-408c-8b4a-e55b3f7d3dc1 kind: Event lastTimestamp: "2026-07-06T20:54:48Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:9b7f9014cc3727e4fd95a945572d3f70017252d596bd8941a1887e4a1514875c" already present on machine metadata: creationTimestamp: "2026-07-06T20:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:54:48Z" name: isvc-primary-f8d41a-predictor-656db64bb8-495pz.18bfcebd0f3f621c namespace: kserve-ci-e2e-test resourceVersion: "19601" uid: 9d0c13bd-3a27-473d-91f0-c2474f46ec6c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:54:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-f8d41a-predictor-656db64bb8-495pz namespace: kserve-ci-e2e-test resourceVersion: "19584" uid: 2badce2e-d8c6-408c-8b4a-e55b3f7d3dc1 kind: Event lastTimestamp: "2026-07-06T20:54:48Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-06T20:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:54:48Z" name: isvc-primary-f8d41a-predictor-656db64bb8-495pz.18bfcebd12944d45 namespace: kserve-ci-e2e-test resourceVersion: "19603" uid: 6a3ea8ef-4eac-465d-8fc8-a737acafdaf0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:54:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-f8d41a-predictor-656db64bb8-495pz namespace: kserve-ci-e2e-test resourceVersion: "19584" uid: 2badce2e-d8c6-408c-8b4a-e55b3f7d3dc1 kind: Event lastTimestamp: "2026-07-06T20:54:48Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-06T20:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:54:48Z" name: isvc-primary-f8d41a-predictor-656db64bb8-495pz.18bfcebd13094b9f namespace: kserve-ci-e2e-test resourceVersion: "19604" uid: 2bf0e47f-9f6b-497c-937f-2b7f61cb8e5e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:54:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-f8d41a-predictor-656db64bb8-495pz namespace: kserve-ci-e2e-test resourceVersion: "19584" uid: 2badce2e-d8c6-408c-8b4a-e55b3f7d3dc1 kind: Event lastTimestamp: "2026-07-06T20:54:53Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1703" already present on machine metadata: creationTimestamp: "2026-07-06T20:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:54:53Z" name: isvc-primary-f8d41a-predictor-656db64bb8-495pz.18bfcebe18bb68cd namespace: kserve-ci-e2e-test resourceVersion: "19642" uid: 108f19bd-3a7d-4f8e-beea-9cd790d38d10 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:54:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-f8d41a-predictor-656db64bb8-495pz namespace: kserve-ci-e2e-test resourceVersion: "19584" uid: 2badce2e-d8c6-408c-8b4a-e55b3f7d3dc1 kind: Event lastTimestamp: "2026-07-06T20:54:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:54:53Z" name: isvc-primary-f8d41a-predictor-656db64bb8-495pz.18bfcebe1c550668 namespace: kserve-ci-e2e-test resourceVersion: "19646" uid: 121a1afe-8517-4ce6-93e0-b9bf856f8b39 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:54:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-f8d41a-predictor-656db64bb8-495pz namespace: kserve-ci-e2e-test resourceVersion: "19584" uid: 2badce2e-d8c6-408c-8b4a-e55b3f7d3dc1 kind: Event lastTimestamp: "2026-07-06T20:54:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:54:53Z" name: isvc-primary-f8d41a-predictor-656db64bb8-495pz.18bfcebe1cce822e namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: 5285c6b2-7f21-442e-b9c0-16359aa997a8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:54:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-f8d41a-predictor-656db64bb8-495pz namespace: kserve-ci-e2e-test resourceVersion: "19584" uid: 2badce2e-d8c6-408c-8b4a-e55b3f7d3dc1 kind: Event lastTimestamp: "2026-07-06T20:54:53Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T20:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:54:53Z" name: isvc-primary-f8d41a-predictor-656db64bb8-495pz.18bfcebe1cd8e1e6 namespace: kserve-ci-e2e-test resourceVersion: "19651" uid: 523b1828-6a22-41df-9e17-6bc5c09a0216 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:54:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-f8d41a-predictor-656db64bb8-495pz namespace: kserve-ci-e2e-test resourceVersion: "19584" uid: 2badce2e-d8c6-408c-8b4a-e55b3f7d3dc1 kind: Event lastTimestamp: "2026-07-06T20:54:53Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:54:53Z" name: isvc-primary-f8d41a-predictor-656db64bb8-495pz.18bfcebe1fad97d6 namespace: kserve-ci-e2e-test resourceVersion: "19652" uid: 2d106471-f01f-4231-9581-681b8d52d76d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:54:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-f8d41a-predictor-656db64bb8-495pz namespace: kserve-ci-e2e-test resourceVersion: "19584" uid: 2badce2e-d8c6-408c-8b4a-e55b3f7d3dc1 kind: Event lastTimestamp: "2026-07-06T20:54:53Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:54:53Z" name: isvc-primary-f8d41a-predictor-656db64bb8-495pz.18bfcebe20235714 namespace: kserve-ci-e2e-test resourceVersion: "19653" uid: 3907b3f3-c920-4002-9421-d93b16b685d6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-06T20:54:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-f8d41a-predictor-656db64bb8-495pz namespace: kserve-ci-e2e-test resourceVersion: "19584" uid: 2badce2e-d8c6-408c-8b4a-e55b3f7d3dc1 kind: Event lastTimestamp: "2026-07-06T20:56:01Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:54:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:01Z" name: isvc-primary-f8d41a-predictor-656db64bb8-495pz.18bfcebe90a9d2a4 namespace: kserve-ci-e2e-test resourceVersion: "20132" uid: 216a53e8-3f3f-4f06-ba27-5edfe0826a93 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-f8d41a-predictor-656db64bb8-495pz namespace: kserve-ci-e2e-test resourceVersion: "19584" uid: 2badce2e-d8c6-408c-8b4a-e55b3f7d3dc1 kind: Event lastTimestamp: "2026-07-06T20:56:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:32Z" name: isvc-primary-f8d41a-predictor-656db64bb8-495pz.18bfced540f57653 namespace: kserve-ci-e2e-test resourceVersion: "20461" uid: 7b3b8583-a765-4dd4-8cb2-7827ae05e83f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-f8d41a-predictor-656db64bb8-495pz namespace: kserve-ci-e2e-test resourceVersion: "19584" uid: 2badce2e-d8c6-408c-8b4a-e55b3f7d3dc1 kind: Event lastTimestamp: "2026-07-06T20:56:32Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:32Z" name: isvc-primary-f8d41a-predictor-656db64bb8-495pz.18bfced540f60312 namespace: kserve-ci-e2e-test resourceVersion: "20462" uid: bf121828-855d-4f24-aaf6-b1444e9d8fe9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-f8d41a-predictor-656db64bb8-495pz namespace: kserve-ci-e2e-test resourceVersion: "19584" uid: 2badce2e-d8c6-408c-8b4a-e55b3f7d3dc1 kind: Event lastTimestamp: "2026-07-06T20:56:36Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:56:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:36Z" name: isvc-primary-f8d41a-predictor-656db64bb8-495pz.18bfced614e92f4b namespace: kserve-ci-e2e-test resourceVersion: "20493" uid: 99ff1a57-ab8b-40b2-9a2c-61a6f63ab8f2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:54:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-f8d41a-predictor-656db64bb8 namespace: kserve-ci-e2e-test resourceVersion: "19573" uid: e4cfe928-d703-4050-89b7-f9d81bed1a3d kind: Event lastTimestamp: "2026-07-06T20:54:48Z" message: 'Created pod: isvc-primary-f8d41a-predictor-656db64bb8-495pz' metadata: creationTimestamp: "2026-07-06T20:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:54:48Z" name: isvc-primary-f8d41a-predictor-656db64bb8.18bfcebcf355988f namespace: kserve-ci-e2e-test resourceVersion: "19585" uid: 02df5c21-1ff0-4a4f-b7f3-c05e58915949 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:54:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-f8d41a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19571" uid: c8c94741-7c91-4e78-879d-1daafc2f7588 kind: Event lastTimestamp: "2026-07-06T20:54:48Z" message: Scaled up replica set isvc-primary-f8d41a-predictor-656db64bb8 from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:54:48Z" name: isvc-primary-f8d41a-predictor.18bfcebcecc7d607 namespace: kserve-ci-e2e-test resourceVersion: "19574" uid: 61d6233a-55b5-4ab5-8065-c129de8c2b07 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:55:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-f8d41a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19572" uid: 92f65b1e-8879-483e-aab4-3c5923d83c31 kind: Event lastTimestamp: "2026-07-06T20:55:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:55:33Z" name: isvc-primary-f8d41a-predictor.18bfcec06be01f43 namespace: kserve-ci-e2e-test resourceVersion: "19935" uid: 7b87a11e-4272-499f-aca1-6650878616c5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:55:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-f8d41a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19572" uid: 92f65b1e-8879-483e-aab4-3c5923d83c31 kind: Event lastTimestamp: "2026-07-06T20:55:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:55:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:55:33Z" name: isvc-primary-f8d41a-predictor.18bfcec06c5a8aa3 namespace: kserve-ci-e2e-test resourceVersion: "19936" uid: 5afac1a8-9459-4e94-93f0-15774343904d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T20:55:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-f8d41a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19722" uid: 92f65b1e-8879-483e-aab4-3c5923d83c31 kind: Event lastTimestamp: "2026-07-06T20:56:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:56:18Z" name: isvc-primary-f8d41a-predictor.18bfcec3eb239b0b namespace: kserve-ci-e2e-test resourceVersion: "20262" uid: c20bc84c-47e0-42b3-a984-5680d8cdecbd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T20:55:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-f8d41a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19722" uid: 92f65b1e-8879-483e-aab4-3c5923d83c31 kind: Event lastTimestamp: "2026-07-06T20:56:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:56:18Z" name: isvc-primary-f8d41a-predictor.18bfcec3eba63943 namespace: kserve-ci-e2e-test resourceVersion: "20264" uid: cf00a042-5566-4e79-87be-8f0d815c8fea reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-06T20:54:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-f8d41a namespace: kserve-ci-e2e-test resourceVersion: "19562" uid: da581b34-2b72-456c-82da-fc42d7fd5086 kind: Event lastTimestamp: "2026-07-06T20:56:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:56:11Z" name: isvc-primary-f8d41a.18bfcebcece68a04 namespace: kserve-ci-e2e-test resourceVersion: "20212" uid: 71d8a346-1858-4dc7-84bb-c54c48f1f6ce reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:54:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-f8d41a namespace: kserve-ci-e2e-test resourceVersion: "19562" uid: da581b34-2b72-456c-82da-fc42d7fd5086 kind: Event lastTimestamp: "2026-07-06T20:54:48Z" message: 'Failed to update status for InferenceService "isvc-primary-f8d41a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-f8d41a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:54:48Z" name: isvc-primary-f8d41a.18bfcebcee492a66 namespace: kserve-ci-e2e-test resourceVersion: "19578" uid: 5393c080-3570-4e74-a547-a13564c7414c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:54:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-f8d41a namespace: kserve-ci-e2e-test resourceVersion: "19562" uid: da581b34-2b72-456c-82da-fc42d7fd5086 kind: Event lastTimestamp: "2026-07-06T20:54:48Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-f8d41a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:54:48Z" name: isvc-primary-f8d41a.18bfcebcee49a07d namespace: kserve-ci-e2e-test resourceVersion: "19580" uid: 22f2956f-e693-4bed-976e-e6e614c84b18 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-f8d41a namespace: kserve-ci-e2e-test resourceVersion: "20209" uid: da581b34-2b72-456c-82da-fc42d7fd5086 kind: Event lastTimestamp: "2026-07-06T20:56:11Z" message: InferenceService [isvc-primary-f8d41a] is Ready metadata: creationTimestamp: "2026-07-06T20:56:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:56:11Z" name: isvc-primary-f8d41a.18bfced049b77b47 namespace: kserve-ci-e2e-test resourceVersion: "20210" uid: 15f1c6a2-b663-42e7-ba7c-ae5ddb628904 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:44:31.566447Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql namespace: kserve-ci-e2e-test resourceVersion: "14312" uid: 7bb31b47-e3de-4e98-b015-7d35be23370e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql to ip-10-0-139-250.ec2.internal metadata: creationTimestamp: "2026-07-06T20:44:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:44:31Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql.18bfce2d55917acd namespace: kserve-ci-e2e-test resourceVersion: "14319" uid: 2a2a940d-c8e6-434d-9d8b-f5013d3536bc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-95cfcc986-db5pj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:44:31Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql namespace: kserve-ci-e2e-test resourceVersion: "14323" uid: 7bb31b47-e3de-4e98-b015-7d35be23370e kind: Event lastTimestamp: "2026-07-06T20:44:31Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:44:31Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql.18bfce2d6ef11449 namespace: kserve-ci-e2e-test resourceVersion: "14332" uid: b09d38aa-70dc-4586-83fd-848d84e3c691 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:44:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: 7bb31b47-e3de-4e98-b015-7d35be23370e kind: Event lastTimestamp: "2026-07-06T20:44:32Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:9b7f9014cc3727e4fd95a945572d3f70017252d596bd8941a1887e4a1514875c" metadata: creationTimestamp: "2026-07-06T20:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:44:32Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql.18bfce2d7088aba7 namespace: kserve-ci-e2e-test resourceVersion: "14334" uid: 56421867-2671-4c27-9c62-019d3dae8d28 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:44:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: 7bb31b47-e3de-4e98-b015-7d35be23370e kind: Event lastTimestamp: "2026-07-06T20:44:36Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:9b7f9014cc3727e4fd95a945572d3f70017252d596bd8941a1887e4a1514875c" in 4.498s (4.498s including waiting). Image size: 300633537 bytes.' metadata: creationTimestamp: "2026-07-06T20:44:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:44:36Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql.18bfce2e7ca55a4a namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 2c99f561-1d20-4d40-a59c-9817942eebaf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:44:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: 7bb31b47-e3de-4e98-b015-7d35be23370e kind: Event lastTimestamp: "2026-07-06T20:44:36Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-06T20:44:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:44:36Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql.18bfce2e7fbb1474 namespace: kserve-ci-e2e-test resourceVersion: "14365" uid: 8df4eff2-cdd2-46e5-8d96-8c9f1ae09d51 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:44:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: 7bb31b47-e3de-4e98-b015-7d35be23370e kind: Event lastTimestamp: "2026-07-06T20:44:36Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-06T20:44:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:44:36Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql.18bfce2e80371490 namespace: kserve-ci-e2e-test resourceVersion: "14366" uid: ffb0ee34-22e6-45d3-b3ef-873c470ffb37 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: 7bb31b47-e3de-4e98-b015-7d35be23370e kind: Event lastTimestamp: "2026-07-06T20:44:40Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1703" metadata: creationTimestamp: "2026-07-06T20:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:44:40Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql.18bfce2f8532e6c5 namespace: kserve-ci-e2e-test resourceVersion: "14401" uid: cded5f61-d1b6-4965-b45b-7a822cf72008 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:44:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: 7bb31b47-e3de-4e98-b015-7d35be23370e kind: Event lastTimestamp: "2026-07-06T20:44:57Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1703" in 16.609s (16.609s including waiting). Image size: 1555746730 bytes.' metadata: creationTimestamp: "2026-07-06T20:44:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:44:57Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql.18bfce33633b8178 namespace: kserve-ci-e2e-test resourceVersion: "14525" uid: a7275308-ff7c-4c3e-a2f1-99b0e7192b5e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:44:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: 7bb31b47-e3de-4e98-b015-7d35be23370e kind: Event lastTimestamp: "2026-07-06T20:44:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:44:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:44:57Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql.18bfce33665e354d namespace: kserve-ci-e2e-test resourceVersion: "14527" uid: 58aec0a9-7650-4487-863a-6f5bf64589fb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:44:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: 7bb31b47-e3de-4e98-b015-7d35be23370e kind: Event lastTimestamp: "2026-07-06T20:44:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:44:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:44:57Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql.18bfce3366d574cb namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: 9dead3e6-03e7-4514-ae0f-a5a6414e8e1d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:44:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: 7bb31b47-e3de-4e98-b015-7d35be23370e kind: Event lastTimestamp: "2026-07-06T20:44:57Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-06T20:44:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:44:57Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql.18bfce3366e40853 namespace: kserve-ci-e2e-test resourceVersion: "14529" uid: 69de8ea8-cf49-43f4-8da2-d47ab687e785 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:45:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: 7bb31b47-e3de-4e98-b015-7d35be23370e kind: Event lastTimestamp: "2026-07-06T20:45:00Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.748s (2.748s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-06T20:45:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:45:00Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql.18bfce340abca354 namespace: kserve-ci-e2e-test resourceVersion: "14545" uid: 24a35b0d-af47-4f61-b657-8dd2defd7629 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:45:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: 7bb31b47-e3de-4e98-b015-7d35be23370e kind: Event lastTimestamp: "2026-07-06T20:45:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:45:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:45:00Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql.18bfce340f55bff2 namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: 871e0af3-97f0-469e-b7ed-3dae4ae6805d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:45:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: 7bb31b47-e3de-4e98-b015-7d35be23370e kind: Event lastTimestamp: "2026-07-06T20:45:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:45:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:45:00Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql.18bfce341125c7e7 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: abede31a-0f46-453b-b21c-a067460cc49c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:45:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: 7bb31b47-e3de-4e98-b015-7d35be23370e kind: Event lastTimestamp: "2026-07-06T20:45:00Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:0706c86054be2f22ee66c8739e97daebd0094b98a978f91cd3fb4f59862bad3f" metadata: creationTimestamp: "2026-07-06T20:45:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:45:00Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql.18bfce34113346cc namespace: kserve-ci-e2e-test resourceVersion: "14550" uid: 8826d4ea-ecb7-42cb-81df-97cec58a5896 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:45:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: 7bb31b47-e3de-4e98-b015-7d35be23370e kind: Event lastTimestamp: "2026-07-06T20:45:03Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:0706c86054be2f22ee66c8739e97daebd0094b98a978f91cd3fb4f59862bad3f" in 3.306s (3.306s including waiting). Image size: 239997029 bytes.' metadata: creationTimestamp: "2026-07-06T20:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:45:03Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql.18bfce34d64f44c4 namespace: kserve-ci-e2e-test resourceVersion: "14571" uid: 0d78b400-b659-4cf3-b929-af013c46de77 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:45:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: 7bb31b47-e3de-4e98-b015-7d35be23370e kind: Event lastTimestamp: "2026-07-06T20:45:03Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-06T20:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:45:03Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql.18bfce34dc1c34ba namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: 941817b8-c031-41c2-8a05-3071f2f814e4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:45:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: 7bb31b47-e3de-4e98-b015-7d35be23370e kind: Event lastTimestamp: "2026-07-06T20:45:03Z" message: Started container agent metadata: creationTimestamp: "2026-07-06T20:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:45:03Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql.18bfce34dd2ef46b namespace: kserve-ci-e2e-test resourceVersion: "14574" uid: 775a935c-ca87-48a5-9c8a-6413552e205d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-06T20:45:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: 7bb31b47-e3de-4e98-b015-7d35be23370e kind: Event lastTimestamp: "2026-07-06T20:46:37Z" message: 'Readiness probe failed: dial tcp 10.134.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:45:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:37Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql.18bfce35221cdec5 namespace: kserve-ci-e2e-test resourceVersion: "15390" uid: c0a39780-e05b-4940-bd47-4b2986d49ad7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-06T20:45:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: 7bb31b47-e3de-4e98-b015-7d35be23370e kind: Event lastTimestamp: "2026-07-06T20:46:37Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-06T20:45:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:37Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql.18bfce352226833a namespace: kserve-ci-e2e-test resourceVersion: "15391" uid: 47d28356-bd0e-46ae-b3a7-74e0d73e1d35 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: 7bb31b47-e3de-4e98-b015-7d35be23370e kind: Event lastTimestamp: "2026-07-06T20:46:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T20:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:26Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql.18bfce482d1fad03 namespace: kserve-ci-e2e-test resourceVersion: "15224" uid: c21d7e85-5910-4213-9a30-73e9d0810da6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: 7bb31b47-e3de-4e98-b015-7d35be23370e kind: Event lastTimestamp: "2026-07-06T20:46:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:26Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql.18bfce482d205fd0 namespace: kserve-ci-e2e-test resourceVersion: "15227" uid: e0454bd4-1497-40bf-af79-424d4eac4234 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: 7bb31b47-e3de-4e98-b015-7d35be23370e kind: Event lastTimestamp: "2026-07-06T20:46:26Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-06T20:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:26Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql.18bfce482d26608d namespace: kserve-ci-e2e-test resourceVersion: "15232" uid: 65752d68-ef58-4935-bb7b-e88101baa526 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:46:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: 7bb31b47-e3de-4e98-b015-7d35be23370e kind: Event lastTimestamp: "2026-07-06T20:46:40Z" message: 'Readiness probe failed: Get "https://10.134.0.20:8643/healthz": dial tcp 10.134.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:46:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:40Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql.18bfce48ec7aa38b namespace: kserve-ci-e2e-test resourceVersion: "15413" uid: f856ff9b-16e7-4a8f-8b31-1acf4cd8c8eb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:44:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5 namespace: kserve-ci-e2e-test resourceVersion: "14302" uid: e6fb02e5-9205-4c04-9e6e-3f16a198d649 kind: Event lastTimestamp: "2026-07-06T20:44:31Z" message: 'Created pod: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5-tvxql' metadata: creationTimestamp: "2026-07-06T20:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:44:31Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5.18bfce2d54ee1280 namespace: kserve-ci-e2e-test resourceVersion: "14313" uid: 0f618ba8-e47c-4a4f-856d-d544f1b27acc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:44:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-ea0d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "14301" uid: 854760bd-8c65-4154-b3e7-049c070f6337 kind: Event lastTimestamp: "2026-07-06T20:44:31Z" message: Scaled up replica set isvc-raw-sklearn-batcher-ea0d4-predictor-f8dcfb6b5 from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:44:31Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor.18bfce2d4eca1a9a namespace: kserve-ci-e2e-test resourceVersion: "14305" uid: 4734c6a0-dfb2-4d80-bf3e-5dc52ff54d57 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:44:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-ea0d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "14311" uid: 12445740-26e7-4a07-8319-2ec0a65c068f kind: Event lastTimestamp: "2026-07-06T20:44:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:44:46Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor.18bfce30d478a7ca namespace: kserve-ci-e2e-test resourceVersion: "14451" uid: aec7cb7c-3886-4b16-86eb-8f173327cc2d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:44:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-ea0d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "14311" uid: 12445740-26e7-4a07-8319-2ec0a65c068f kind: Event lastTimestamp: "2026-07-06T20:44:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:44:46Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor.18bfce30d4f4cf65 namespace: kserve-ci-e2e-test resourceVersion: "14452" uid: a20347e3-eea3-4c8c-a700-fc7d68e8b111 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:45:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-ea0d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "14450" uid: 12445740-26e7-4a07-8319-2ec0a65c068f kind: Event lastTimestamp: "2026-07-06T20:45:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: unable to fetch metrics from resource metrics API: an error on the server ("Internal Server Error: \"/apis/metrics.k8s.io/v1beta1/namespaces/kserve-ci-e2e-test/pods?labelSelector=app%3Disvc.isvc-raw-sklearn-batcher-ea0d4-predictor\": Post \"https://172.31.0.1:443/apis/authorization.k8s.io/v1/subjectaccessreviews?timeout=10s\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)") has prevented the request from succeeding (get pods.metrics.k8s.io)' metadata: creationTimestamp: "2026-07-06T20:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:45:36Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor.18bfce36a83fa1c7 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: a347bb80-cd2c-4ebf-a646-6c5756efa59f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:45:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-ea0d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "14450" uid: 12445740-26e7-4a07-8319-2ec0a65c068f kind: Event lastTimestamp: "2026-07-06T20:45:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: unable to fetch metrics from resource metrics API: an error on the server ("Internal Server Error: \"/apis/metrics.k8s.io/v1beta1/namespaces/kserve-ci-e2e-test/pods?labelSelector=app%3Disvc.isvc-raw-sklearn-batcher-ea0d4-predictor\": Post \"https://172.31.0.1:443/apis/authorization.k8s.io/v1/subjectaccessreviews?timeout=10s\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)") has prevented the request from succeeding (get pods.metrics.k8s.io)' metadata: creationTimestamp: "2026-07-06T20:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:45:36Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor.18bfce36a8bf18b5 namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: 28df57da-5820-44f2-83f9-bb362d717ce7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:45:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-ea0d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "14639" uid: 12445740-26e7-4a07-8319-2ec0a65c068f kind: Event lastTimestamp: "2026-07-06T20:46:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:45:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:46:15Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor.18bfce3ea6a74173 namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: f30b36ed-e57f-45f7-83b5-8a5518aa9b0d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:45:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-ea0d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "14639" uid: 12445740-26e7-4a07-8319-2ec0a65c068f kind: Event lastTimestamp: "2026-07-06T20:46:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:45:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:46:15Z" name: isvc-raw-sklearn-batcher-ea0d4-predictor.18bfce3ea717a11e namespace: kserve-ci-e2e-test resourceVersion: "15096" uid: 4b03ca60-4dc6-410e-9145-3e2da8fe5e74 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-06T20:44:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-ea0d4 namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: bcbbf4ed-668b-40e9-8fa4-3d8a4adbdb80 kind: Event lastTimestamp: "2026-07-06T20:46:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:46:17Z" name: isvc-raw-sklearn-batcher-ea0d4.18bfce2d5502c7e3 namespace: kserve-ci-e2e-test resourceVersion: "15113" uid: 3674ff78-94a8-46a8-95c6-b3ef7bd0636f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:44:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-ea0d4 namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: bcbbf4ed-668b-40e9-8fa4-3d8a4adbdb80 kind: Event lastTimestamp: "2026-07-06T20:44:31Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-ea0d4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-ea0d4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:44:31Z" name: isvc-raw-sklearn-batcher-ea0d4.18bfce2d55743568 namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: f33a5d23-c441-426d-ab7d-913dc1ba0247 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:44:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-ea0d4 namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: bcbbf4ed-668b-40e9-8fa4-3d8a4adbdb80 kind: Event lastTimestamp: "2026-07-06T20:44:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-ea0d4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:44:31Z" name: isvc-raw-sklearn-batcher-ea0d4.18bfce2d5574a01b namespace: kserve-ci-e2e-test resourceVersion: "14321" uid: f619dd94-7a0d-49e4-9595-f96219cd164c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-ea0d4 namespace: kserve-ci-e2e-test resourceVersion: "15110" uid: bcbbf4ed-668b-40e9-8fa4-3d8a4adbdb80 kind: Event lastTimestamp: "2026-07-06T20:46:17Z" message: InferenceService [isvc-raw-sklearn-batcher-ea0d4] is Ready metadata: creationTimestamp: "2026-07-06T20:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:46:17Z" name: isvc-raw-sklearn-batcher-ea0d4.18bfce45ee15c8e6 namespace: kserve-ci-e2e-test resourceVersion: "15111" uid: 73d9c7b3-3a12-486f-b43b-417dd02eab85 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:56:18.614901Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-f8d41a-predictor-655bb967ff-8f9j6 namespace: kserve-ci-e2e-test resourceVersion: "20286" uid: 47e6f04f-745d-4ace-8dd1-477f950a7aaf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-f8d41a-predictor-655bb967ff-8f9j6 to ip-10-0-139-250.ec2.internal metadata: creationTimestamp: "2026-07-06T20:56:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:56:18Z" name: isvc-secondary-f8d41a-predictor-655bb967ff-8f9j6.18bfced1f4f0bc9e namespace: kserve-ci-e2e-test resourceVersion: "20290" uid: f8c2969a-b65d-4f0b-ac74-66412d41eff4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-95cfcc986-db5pj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:19Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-f8d41a-predictor-655bb967ff-8f9j6 namespace: kserve-ci-e2e-test resourceVersion: "20295" uid: 47e6f04f-745d-4ace-8dd1-477f950a7aaf kind: Event lastTimestamp: "2026-07-06T20:56:19Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:56:19Z" name: isvc-secondary-f8d41a-predictor-655bb967ff-8f9j6.18bfced20ecc6f8a namespace: kserve-ci-e2e-test resourceVersion: "20307" uid: 1138ea96-2a6e-4fd1-8e42-95a6eea65487 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:56:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-f8d41a-predictor-655bb967ff-8f9j6 namespace: kserve-ci-e2e-test resourceVersion: "20288" uid: 47e6f04f-745d-4ace-8dd1-477f950a7aaf kind: Event lastTimestamp: "2026-07-06T20:56:21Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:9b7f9014cc3727e4fd95a945572d3f70017252d596bd8941a1887e4a1514875c" already present on machine metadata: creationTimestamp: "2026-07-06T20:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:21Z" name: isvc-secondary-f8d41a-predictor-655bb967ff-8f9j6.18bfced210664a4f namespace: kserve-ci-e2e-test resourceVersion: "20335" uid: c3e33260-7c0f-4efb-8641-6c7a4f126617 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:56:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-f8d41a-predictor-655bb967ff-8f9j6 namespace: kserve-ci-e2e-test resourceVersion: "20288" uid: 47e6f04f-745d-4ace-8dd1-477f950a7aaf kind: Event lastTimestamp: "2026-07-06T20:56:21Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-06T20:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:21Z" name: isvc-secondary-f8d41a-predictor-655bb967ff-8f9j6.18bfced213118f20 namespace: kserve-ci-e2e-test resourceVersion: "20337" uid: d65007ac-950a-4199-a980-fb629862d327 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:56:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-f8d41a-predictor-655bb967ff-8f9j6 namespace: kserve-ci-e2e-test resourceVersion: "20288" uid: 47e6f04f-745d-4ace-8dd1-477f950a7aaf kind: Event lastTimestamp: "2026-07-06T20:56:21Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-06T20:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:21Z" name: isvc-secondary-f8d41a-predictor-655bb967ff-8f9j6.18bfced2138ae5ef namespace: kserve-ci-e2e-test resourceVersion: "20338" uid: c0ac289c-5150-49f2-8274-563e0a34119b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-f8d41a-predictor-655bb967ff-8f9j6 namespace: kserve-ci-e2e-test resourceVersion: "20288" uid: 47e6f04f-745d-4ace-8dd1-477f950a7aaf kind: Event lastTimestamp: "2026-07-06T20:56:24Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-f8d41a-predictor-655bb967ff-8f9j6_kserve-ci-e2e-test(47e6f04f-745d-4ace-8dd1-477f950a7aaf) metadata: creationTimestamp: "2026-07-06T20:56:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:24Z" name: isvc-secondary-f8d41a-predictor-655bb967ff-8f9j6.18bfced35e18e3de namespace: kserve-ci-e2e-test resourceVersion: "20365" uid: a0f2e5df-421e-4c2e-9620-42a46dcc6b32 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-f8d41a-predictor-655bb967ff namespace: kserve-ci-e2e-test resourceVersion: "20277" uid: 107bf4a0-f885-4d4a-8df3-10a7319118e8 kind: Event lastTimestamp: "2026-07-06T20:56:18Z" message: 'Created pod: isvc-secondary-f8d41a-predictor-655bb967ff-8f9j6' metadata: creationTimestamp: "2026-07-06T20:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:56:18Z" name: isvc-secondary-f8d41a-predictor-655bb967ff.18bfced1f471befc namespace: kserve-ci-e2e-test resourceVersion: "20287" uid: ae586e14-563f-4bd1-956a-2a17aaa9618a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-f8d41a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20275" uid: 5bae5ba5-24f6-429e-be81-f2f41cb56ecb kind: Event lastTimestamp: "2026-07-06T20:56:18Z" message: Scaled up replica set isvc-secondary-f8d41a-predictor-655bb967ff from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:56:18Z" name: isvc-secondary-f8d41a-predictor.18bfced1ef03eb94 namespace: kserve-ci-e2e-test resourceVersion: "20278" uid: 00eddcf0-605c-4bcf-b3b9-34f66063ab2a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-07-06T20:56:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-f8d41a namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: 8dc35f5e-d4f3-4adc-80c1-6476fbfb7f43 kind: Event lastTimestamp: "2026-07-06T20:56:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:56:24Z" name: isvc-secondary-f8d41a.18bfced1ef1841ab namespace: kserve-ci-e2e-test resourceVersion: "20371" uid: 87d58f03-c668-4eaa-86ff-b1d7685ee8d9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:56:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-f8d41a namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: 8dc35f5e-d4f3-4adc-80c1-6476fbfb7f43 kind: Event lastTimestamp: "2026-07-06T20:56:18Z" message: 'Failed to update status for InferenceService "isvc-secondary-f8d41a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-f8d41a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:56:18Z" name: isvc-secondary-f8d41a.18bfced1ef90a0a9 namespace: kserve-ci-e2e-test resourceVersion: "20296" uid: 73704eb2-a5ba-4a1d-8d1c-1bed84c4e75c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:56:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-f8d41a namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: 8dc35f5e-d4f3-4adc-80c1-6476fbfb7f43 kind: Event lastTimestamp: "2026-07-06T20:56:18Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-f8d41a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:56:18Z" name: isvc-secondary-f8d41a.18bfced1ef90ef89 namespace: kserve-ci-e2e-test resourceVersion: "20297" uid: 52f587b7-1b5a-475d-8c3c-d3364f8817ee reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:46:26.865229Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj namespace: kserve-ci-e2e-test resourceVersion: "15215" uid: 0e2375f4-5834-491d-b5c8-4cfb0f3145da kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj to ip-10-0-139-250.ec2.internal metadata: creationTimestamp: "2026-07-06T20:46:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:46:26Z" name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj.18bfce482de94907 namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: 10ecfdda-545f-4937-933d-4b93b823f616 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-95cfcc986-db5pj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:27Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj namespace: kserve-ci-e2e-test resourceVersion: "15241" uid: 0e2375f4-5834-491d-b5c8-4cfb0f3145da kind: Event lastTimestamp: "2026-07-06T20:46:27Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:46:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:46:27Z" name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj.18bfce48476a2b97 namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: d8efea1b-dbd7-488a-a132-8c0da630ff9a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 0e2375f4-5834-491d-b5c8-4cfb0f3145da kind: Event lastTimestamp: "2026-07-06T20:46:27Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:9b7f9014cc3727e4fd95a945572d3f70017252d596bd8941a1887e4a1514875c" already present on machine metadata: creationTimestamp: "2026-07-06T20:46:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:27Z" name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj.18bfce48491988b9 namespace: kserve-ci-e2e-test resourceVersion: "15276" uid: 35a41c14-b3d9-4bf5-9575-b643b85c7941 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 0e2375f4-5834-491d-b5c8-4cfb0f3145da kind: Event lastTimestamp: "2026-07-06T20:46:27Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-06T20:46:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:27Z" name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj.18bfce484bbe1c09 namespace: kserve-ci-e2e-test resourceVersion: "15277" uid: 26d74921-cac4-4314-8b25-16f0a6fe8c12 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 0e2375f4-5834-491d-b5c8-4cfb0f3145da kind: Event lastTimestamp: "2026-07-06T20:46:27Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-06T20:46:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:27Z" name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj.18bfce484c238edc namespace: kserve-ci-e2e-test resourceVersion: "15278" uid: 22e1de2f-b059-4acd-a0ee-e6e540ee8412 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 0e2375f4-5834-491d-b5c8-4cfb0f3145da kind: Event lastTimestamp: "2026-07-06T20:46:31Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1703" already present on machine metadata: creationTimestamp: "2026-07-06T20:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:31Z" name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj.18bfce493b0cfd71 namespace: kserve-ci-e2e-test resourceVersion: "15336" uid: 863be429-7eb4-4084-acda-a982bb13c4ce reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 0e2375f4-5834-491d-b5c8-4cfb0f3145da kind: Event lastTimestamp: "2026-07-06T20:46:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:31Z" name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj.18bfce493e577ede namespace: kserve-ci-e2e-test resourceVersion: "15339" uid: fa5309e4-a8e6-46af-8e7f-824a3730296c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 0e2375f4-5834-491d-b5c8-4cfb0f3145da kind: Event lastTimestamp: "2026-07-06T20:46:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:31Z" name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj.18bfce493ebb0607 namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: 10be551f-99b4-4cac-acc3-871690aed02c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 0e2375f4-5834-491d-b5c8-4cfb0f3145da kind: Event lastTimestamp: "2026-07-06T20:46:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T20:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:31Z" name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj.18bfce493ec6c6aa namespace: kserve-ci-e2e-test resourceVersion: "15341" uid: 698f24a9-0ecf-4a06-80f0-edf6ee82bed9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 0e2375f4-5834-491d-b5c8-4cfb0f3145da kind: Event lastTimestamp: "2026-07-06T20:46:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:31Z" name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj.18bfce49422a10f7 namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: 656cfdaf-576d-4f8f-935e-beae8b5c4016 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 0e2375f4-5834-491d-b5c8-4cfb0f3145da kind: Event lastTimestamp: "2026-07-06T20:46:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:31Z" name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj.18bfce494291404a namespace: kserve-ci-e2e-test resourceVersion: "15346" uid: 1a360c24-d3b5-4efa-8a5d-aebc74a898e3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-06T20:46:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 0e2375f4-5834-491d-b5c8-4cfb0f3145da kind: Event lastTimestamp: "2026-07-06T20:48:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:46:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:18Z" name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj.18bfce49774ea7ac namespace: kserve-ci-e2e-test resourceVersion: "16332" uid: eefa4f16-1595-40ff-96f9-1aa39c775ac6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 0e2375f4-5834-491d-b5c8-4cfb0f3145da kind: Event lastTimestamp: "2026-07-06T20:48:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T20:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:17Z" name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj.18bfce61f13eaa71 namespace: kserve-ci-e2e-test resourceVersion: "16274" uid: 621d121b-d30c-438b-bae9-044c1491369a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 0e2375f4-5834-491d-b5c8-4cfb0f3145da kind: Event lastTimestamp: "2026-07-06T20:48:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:17Z" name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj.18bfce61f13f37a1 namespace: kserve-ci-e2e-test resourceVersion: "16276" uid: c83679fe-5b07-4f59-9797-bb07ef821af8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 0e2375f4-5834-491d-b5c8-4cfb0f3145da kind: Event lastTimestamp: "2026-07-06T20:48:18Z" message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": dial tcp 10.134.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:18Z" name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj.18bfce6225ae5ca2 namespace: kserve-ci-e2e-test resourceVersion: "16331" uid: 6445d0f1-b81c-45d5-b1a5-b8782c0d6a77 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5 namespace: kserve-ci-e2e-test resourceVersion: "15190" uid: ed67ce5d-a797-4791-bf9e-56e1521eb57e kind: Event lastTimestamp: "2026-07-06T20:46:26Z" message: 'Created pod: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5-4wtfj' metadata: creationTimestamp: "2026-07-06T20:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:46:26Z" name: isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5.18bfce482c70194e namespace: kserve-ci-e2e-test resourceVersion: "15220" uid: 8e50c766-8600-44eb-bd20-0fa99b0abaf9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-b4160-predictor namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: caf14ef5-7a20-449d-9c47-e99682508a2d kind: Event lastTimestamp: "2026-07-06T20:46:26Z" message: Scaled up replica set isvc-sklearn-graph-raw-b4160-predictor-bdd664fb5 from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:46:26Z" name: isvc-sklearn-graph-raw-b4160-predictor.18bfce4825a773b4 namespace: kserve-ci-e2e-test resourceVersion: "15192" uid: 58bc9d4a-e231-42b9-bf4b-d05ccc74d176 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:46:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-b4160-predictor namespace: kserve-ci-e2e-test resourceVersion: "15189" uid: f1dbb1ae-4054-4525-b380-4d733a4c32a0 kind: Event lastTimestamp: "2026-07-06T20:46:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:46:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:46:56Z" name: isvc-sklearn-graph-raw-b4160-predictor.18bfce4ba4883e64 namespace: kserve-ci-e2e-test resourceVersion: "15547" uid: de23a292-6f47-47db-9ffa-5df3d746f24a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:46:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-b4160-predictor namespace: kserve-ci-e2e-test resourceVersion: "15189" uid: f1dbb1ae-4054-4525-b380-4d733a4c32a0 kind: Event lastTimestamp: "2026-07-06T20:46:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:46:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:46:56Z" name: isvc-sklearn-graph-raw-b4160-predictor.18bfce4ba4e9f269 namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: 1b174f7c-409c-4fe0-bb12-a375537a2f55 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-06T20:47:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-b4160-predictor namespace: kserve-ci-e2e-test resourceVersion: "15425" uid: f1dbb1ae-4054-4525-b380-4d733a4c32a0 kind: Event lastTimestamp: "2026-07-06T20:48:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:47:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:11Z" name: isvc-sklearn-graph-raw-b4160-predictor.18bfce52a39c0c50 namespace: kserve-ci-e2e-test resourceVersion: "16138" uid: 136428e3-03bb-4dfc-8c0e-4a02ac557ac4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-06T20:47:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-b4160-predictor namespace: kserve-ci-e2e-test resourceVersion: "15425" uid: f1dbb1ae-4054-4525-b380-4d733a4c32a0 kind: Event lastTimestamp: "2026-07-06T20:48:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:47:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:11Z" name: isvc-sklearn-graph-raw-b4160-predictor.18bfce52a4143672 namespace: kserve-ci-e2e-test resourceVersion: "16139" uid: b3ad209d-c063-45e1-b079-cb5a4023c29e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-06T20:46:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-b4160 namespace: kserve-ci-e2e-test resourceVersion: "15178" uid: 5343ce35-82d7-44c5-979a-09b785c6c354 kind: Event lastTimestamp: "2026-07-06T20:47:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:47:48Z" name: isvc-sklearn-graph-raw-b4160.18bfce4825d41d2e namespace: kserve-ci-e2e-test resourceVersion: "15935" uid: 9e2416d8-db0a-485e-ba70-07b39b890814 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-b4160 namespace: kserve-ci-e2e-test resourceVersion: "15178" uid: 5343ce35-82d7-44c5-979a-09b785c6c354 kind: Event lastTimestamp: "2026-07-06T20:46:26Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-b4160": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-b4160": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:46:26Z" name: isvc-sklearn-graph-raw-b4160.18bfce4826880022 namespace: kserve-ci-e2e-test resourceVersion: "15195" uid: 5113fe30-c3e0-4790-9acc-06ea3e444425 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-b4160 namespace: kserve-ci-e2e-test resourceVersion: "15933" uid: 5343ce35-82d7-44c5-979a-09b785c6c354 kind: Event lastTimestamp: "2026-07-06T20:47:48Z" message: InferenceService [isvc-sklearn-graph-raw-b4160] is Ready metadata: creationTimestamp: "2026-07-06T20:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:47:48Z" name: isvc-sklearn-graph-raw-b4160.18bfce5b301a2e5f namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: 0c7f55d9-0223-4de8-9013-effed19bd5e0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:48:17.402920Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn namespace: kserve-ci-e2e-test resourceVersion: "16232" uid: fa27e854-5a03-43bf-b4d9-bc2f9fcf1d05 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn to ip-10-0-139-250.ec2.internal metadata: creationTimestamp: "2026-07-06T20:48:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:48:17Z" name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn.18bfce61ea789c40 namespace: kserve-ci-e2e-test resourceVersion: "16243" uid: 59a44976-b514-4a8a-9a0c-722c82824063 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-95cfcc986-db5pj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:17Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn namespace: kserve-ci-e2e-test resourceVersion: "16262" uid: fa27e854-5a03-43bf-b4d9-bc2f9fcf1d05 kind: Event lastTimestamp: "2026-07-06T20:48:17Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:48:17Z" name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn.18bfce6204deb06a namespace: kserve-ci-e2e-test resourceVersion: "16313" uid: 9dff95d7-c473-4fbe-a410-b2f817068caa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: fa27e854-5a03-43bf-b4d9-bc2f9fcf1d05 kind: Event lastTimestamp: "2026-07-06T20:48:17Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:9b7f9014cc3727e4fd95a945572d3f70017252d596bd8941a1887e4a1514875c" already present on machine metadata: creationTimestamp: "2026-07-06T20:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:17Z" name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn.18bfce6206e7e03a namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: d3c4a27c-8038-4137-85d1-b845f873a758 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: fa27e854-5a03-43bf-b4d9-bc2f9fcf1d05 kind: Event lastTimestamp: "2026-07-06T20:48:17Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-06T20:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:17Z" name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn.18bfce6209ea00ee namespace: kserve-ci-e2e-test resourceVersion: "16319" uid: 833a35ec-5b44-4b28-961d-1832501ac814 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: fa27e854-5a03-43bf-b4d9-bc2f9fcf1d05 kind: Event lastTimestamp: "2026-07-06T20:48:17Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-06T20:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:17Z" name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn.18bfce620a806168 namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 3413c84b-d5d2-438e-863b-0aa4a666cdf8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: fa27e854-5a03-43bf-b4d9-bc2f9fcf1d05 kind: Event lastTimestamp: "2026-07-06T20:48:21Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1703" already present on machine metadata: creationTimestamp: "2026-07-06T20:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:21Z" name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn.18bfce62f24d9ad9 namespace: kserve-ci-e2e-test resourceVersion: "16373" uid: 70790b96-c42b-43d6-90a5-230fd7e24cd1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: fa27e854-5a03-43bf-b4d9-bc2f9fcf1d05 kind: Event lastTimestamp: "2026-07-06T20:48:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:21Z" name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn.18bfce62f56bd5b3 namespace: kserve-ci-e2e-test resourceVersion: "16379" uid: 9e325429-43ae-4d49-bcd2-785639915c58 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: fa27e854-5a03-43bf-b4d9-bc2f9fcf1d05 kind: Event lastTimestamp: "2026-07-06T20:48:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:21Z" name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn.18bfce62f5df3833 namespace: kserve-ci-e2e-test resourceVersion: "16383" uid: b0fa158c-8b7d-481d-a4bf-a9362ff4c831 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: fa27e854-5a03-43bf-b4d9-bc2f9fcf1d05 kind: Event lastTimestamp: "2026-07-06T20:48:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T20:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:21Z" name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn.18bfce62f5e7160c namespace: kserve-ci-e2e-test resourceVersion: "16386" uid: 607e239d-3cca-4e0f-b1a6-15f7f068d22c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: fa27e854-5a03-43bf-b4d9-bc2f9fcf1d05 kind: Event lastTimestamp: "2026-07-06T20:48:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:21Z" name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn.18bfce62f864940f namespace: kserve-ci-e2e-test resourceVersion: "16387" uid: a0190420-a683-4e84-ad3f-ccac82509f57 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: fa27e854-5a03-43bf-b4d9-bc2f9fcf1d05 kind: Event lastTimestamp: "2026-07-06T20:48:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:21Z" name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn.18bfce62f8ccab4a namespace: kserve-ci-e2e-test resourceVersion: "16389" uid: ef220a01-0c52-4f83-8e63-68a8b16c6a9f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-06T20:48:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: fa27e854-5a03-43bf-b4d9-bc2f9fcf1d05 kind: Event lastTimestamp: "2026-07-06T20:50:09Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:48:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:09Z" name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn.18bfce636a793ffb namespace: kserve-ci-e2e-test resourceVersion: "17349" uid: 3fbd8a1c-b94e-4096-ad57-adcb751ea34d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: fa27e854-5a03-43bf-b4d9-bc2f9fcf1d05 kind: Event lastTimestamp: "2026-07-06T20:50:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T20:50:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:07Z" name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn.18bfce7b9cba7b8e namespace: kserve-ci-e2e-test resourceVersion: "17292" uid: 7ee8a325-52ba-4210-bbd9-16754b3129be reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: fa27e854-5a03-43bf-b4d9-bc2f9fcf1d05 kind: Event lastTimestamp: "2026-07-06T20:50:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:50:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:07Z" name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn.18bfce7b9cbbb9e0 namespace: kserve-ci-e2e-test resourceVersion: "17296" uid: abf87145-5b6c-4ae7-918b-337e99384d39 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: fa27e854-5a03-43bf-b4d9-bc2f9fcf1d05 kind: Event lastTimestamp: "2026-07-06T20:50:09Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:50:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:09Z" name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn.18bfce7c18cfeddd namespace: kserve-ci-e2e-test resourceVersion: "17348" uid: 5cbdaf83-f4d1-4010-bac3-2c2d8f9169a4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c namespace: kserve-ci-e2e-test resourceVersion: "16219" uid: 999d2659-594a-42b0-b0f4-ba66e4641d21 kind: Event lastTimestamp: "2026-07-06T20:48:17Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c-276xn' metadata: creationTimestamp: "2026-07-06T20:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:17Z" name: isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c.18bfce61e95e4f84 namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: a27d0c4c-8429-4693-b8bb-d8f3c45fa0e8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-88d19-predictor namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: 740a4299-95c2-4d84-ae47-32f3a465bd3b kind: Event lastTimestamp: "2026-07-06T20:48:17Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-88d19-predictor-86cbcdf9c from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:17Z" name: isvc-sklearn-graph-raw-hpa-88d19-predictor.18bfce61e0bd1421 namespace: kserve-ci-e2e-test resourceVersion: "16221" uid: db749d1a-819f-4739-ab83-b5b230f8f180 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T20:48:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-88d19-predictor namespace: kserve-ci-e2e-test resourceVersion: "16218" uid: 03ecbf4d-ed82-4119-bf9d-2967d06f5c06 kind: Event lastTimestamp: "2026-07-06T20:49:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:48:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:49:17Z" name: isvc-sklearn-graph-raw-hpa-88d19-predictor.18bfce656048e1c0 namespace: kserve-ci-e2e-test resourceVersion: "16812" uid: 9cbd3543-a604-435d-9ef4-fce02af72c6f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T20:48:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-88d19-predictor namespace: kserve-ci-e2e-test resourceVersion: "16218" uid: 03ecbf4d-ed82-4119-bf9d-2967d06f5c06 kind: Event lastTimestamp: "2026-07-06T20:49:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:48:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:49:17Z" name: isvc-sklearn-graph-raw-hpa-88d19-predictor.18bfce6560b2e5fd namespace: kserve-ci-e2e-test resourceVersion: "16813" uid: d6d24c5a-ef6b-4236-84ce-4dc1c13a275c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:49:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-88d19-predictor namespace: kserve-ci-e2e-test resourceVersion: "16474" uid: 03ecbf4d-ed82-4119-bf9d-2967d06f5c06 kind: Event lastTimestamp: "2026-07-06T20:50:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:49:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:50:02Z" name: isvc-sklearn-graph-raw-hpa-88d19-predictor.18bfce735e0129a7 namespace: kserve-ci-e2e-test resourceVersion: "17191" uid: 3b2d66b5-37cd-445a-a307-49a3afbd87c5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:49:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-88d19-predictor namespace: kserve-ci-e2e-test resourceVersion: "16474" uid: 03ecbf4d-ed82-4119-bf9d-2967d06f5c06 kind: Event lastTimestamp: "2026-07-06T20:50:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:49:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:50:02Z" name: isvc-sklearn-graph-raw-hpa-88d19-predictor.18bfce735e7ca8bc namespace: kserve-ci-e2e-test resourceVersion: "17192" uid: 8e9f7e0e-3354-4fb0-9ccc-855c85401bc9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-06T20:48:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-88d19 namespace: kserve-ci-e2e-test resourceVersion: "16206" uid: 4e98f71a-b3b7-4599-a86f-7bc3be0d132d kind: Event lastTimestamp: "2026-07-06T20:49:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:49:40Z" name: isvc-sklearn-graph-raw-hpa-88d19.18bfce61e1b9e46d namespace: kserve-ci-e2e-test resourceVersion: "17003" uid: e9c9a843-ae12-4294-adba-8cd7332a35e0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-88d19 namespace: kserve-ci-e2e-test resourceVersion: "16206" uid: 4e98f71a-b3b7-4599-a86f-7bc3be0d132d kind: Event lastTimestamp: "2026-07-06T20:48:17Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-88d19": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-88d19": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:48:17Z" name: isvc-sklearn-graph-raw-hpa-88d19.18bfce61e288b1dd namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: 25e8a4db-2d98-42ea-a5d6-a4787e19ed65 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:49:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-88d19 namespace: kserve-ci-e2e-test resourceVersion: "17000" uid: 4e98f71a-b3b7-4599-a86f-7bc3be0d132d kind: Event lastTimestamp: "2026-07-06T20:49:40Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-88d19] is Ready metadata: creationTimestamp: "2026-07-06T20:49:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:49:40Z" name: isvc-sklearn-graph-raw-hpa-88d19.18bfce7526315308 namespace: kserve-ci-e2e-test resourceVersion: "17001" uid: d674191d-cf4d-481c-8289-753d5bfc1593 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:51:53.137744Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg namespace: kserve-ci-e2e-test resourceVersion: "18226" uid: e737beeb-91f7-4517-9449-72b17679b380 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg to ip-10-0-139-250.ec2.internal metadata: creationTimestamp: "2026-07-06T20:51:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:51:53Z" name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg.18bfce942544cc62 namespace: kserve-ci-e2e-test resourceVersion: "18233" uid: 3ae9f3e5-f2c0-46de-bbb6-f437ade48c93 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-95cfcc986-db5pj source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:51:53Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg namespace: kserve-ci-e2e-test resourceVersion: "18228" uid: e737beeb-91f7-4517-9449-72b17679b380 kind: Event lastTimestamp: "2026-07-06T20:51:53Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-scale-raw-26dd6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-06T20:51:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:51:53Z" name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg.18bfce942cba5ee2 namespace: kserve-ci-e2e-test resourceVersion: "18269" uid: a918138c-5543-45ee-85a9-eec7ff1589ba reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:51:55Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg namespace: kserve-ci-e2e-test resourceVersion: "18256" uid: e737beeb-91f7-4517-9449-72b17679b380 kind: Event lastTimestamp: "2026-07-06T20:51:55Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:51:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:51:55Z" name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg.18bfce949919b2b5 namespace: kserve-ci-e2e-test resourceVersion: "18283" uid: ce60ead8-cc99-4cf8-9290-1b1fe68077a3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:51:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg namespace: kserve-ci-e2e-test resourceVersion: "18228" uid: e737beeb-91f7-4517-9449-72b17679b380 kind: Event lastTimestamp: "2026-07-06T20:51:55Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:9b7f9014cc3727e4fd95a945572d3f70017252d596bd8941a1887e4a1514875c" already present on machine metadata: creationTimestamp: "2026-07-06T20:51:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:51:55Z" name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg.18bfce949aecbba9 namespace: kserve-ci-e2e-test resourceVersion: "18286" uid: ab50b79e-dd55-4a6b-9d7e-97acfa28e862 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:51:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg namespace: kserve-ci-e2e-test resourceVersion: "18228" uid: e737beeb-91f7-4517-9449-72b17679b380 kind: Event lastTimestamp: "2026-07-06T20:51:55Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-06T20:51:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:51:55Z" name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg.18bfce949e9af573 namespace: kserve-ci-e2e-test resourceVersion: "18288" uid: 64ed1249-0ae0-4c2d-8192-2147ba395df7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:51:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg namespace: kserve-ci-e2e-test resourceVersion: "18228" uid: e737beeb-91f7-4517-9449-72b17679b380 kind: Event lastTimestamp: "2026-07-06T20:51:55Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-06T20:51:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:51:55Z" name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg.18bfce949f12e990 namespace: kserve-ci-e2e-test resourceVersion: "18289" uid: 3ab6563f-ebd5-46de-be6a-1b5195a9eb21 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:51:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg namespace: kserve-ci-e2e-test resourceVersion: "18228" uid: e737beeb-91f7-4517-9449-72b17679b380 kind: Event lastTimestamp: "2026-07-06T20:51:59Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1703" already present on machine metadata: creationTimestamp: "2026-07-06T20:51:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:51:59Z" name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg.18bfce95ab4ec635 namespace: kserve-ci-e2e-test resourceVersion: "18326" uid: 8ddd2aac-1ff6-4456-9cb1-42a0558a279f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:51:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg namespace: kserve-ci-e2e-test resourceVersion: "18228" uid: e737beeb-91f7-4517-9449-72b17679b380 kind: Event lastTimestamp: "2026-07-06T20:51:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:51:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:51:59Z" name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg.18bfce95ae14b236 namespace: kserve-ci-e2e-test resourceVersion: "18328" uid: e15d6eba-f89c-4e2c-8ef1-516f5d091888 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:51:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg namespace: kserve-ci-e2e-test resourceVersion: "18228" uid: e737beeb-91f7-4517-9449-72b17679b380 kind: Event lastTimestamp: "2026-07-06T20:51:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:51:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:51:59Z" name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg.18bfce95ae8bfa1b namespace: kserve-ci-e2e-test resourceVersion: "18329" uid: 2d91c5b4-8cd3-4329-933e-6aac8067d44b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:51:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg namespace: kserve-ci-e2e-test resourceVersion: "18228" uid: e737beeb-91f7-4517-9449-72b17679b380 kind: Event lastTimestamp: "2026-07-06T20:51:59Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T20:51:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:51:59Z" name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg.18bfce95ae9778b4 namespace: kserve-ci-e2e-test resourceVersion: "18330" uid: bb6bf566-a993-483d-b413-2434b57bc5f8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:51:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg namespace: kserve-ci-e2e-test resourceVersion: "18228" uid: e737beeb-91f7-4517-9449-72b17679b380 kind: Event lastTimestamp: "2026-07-06T20:51:59Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:51:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:51:59Z" name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg.18bfce95b2077358 namespace: kserve-ci-e2e-test resourceVersion: "18332" uid: 1f3ec526-e45a-44a5-9eb2-eb2f28bc7eba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:51:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg namespace: kserve-ci-e2e-test resourceVersion: "18228" uid: e737beeb-91f7-4517-9449-72b17679b380 kind: Event lastTimestamp: "2026-07-06T20:51:59Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:51:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:51:59Z" name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg.18bfce95b27a30bb namespace: kserve-ci-e2e-test resourceVersion: "18334" uid: cc30fdc9-b9d1-4d12-95a0-199303abe481 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 18 eventTime: null firstTimestamp: "2026-07-06T20:52:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg namespace: kserve-ci-e2e-test resourceVersion: "18228" uid: e737beeb-91f7-4517-9449-72b17679b380 kind: Event lastTimestamp: "2026-07-06T20:54:43Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:52:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:54:43Z" name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg.18bfce962320cb89 namespace: kserve-ci-e2e-test resourceVersion: "19524" uid: 2df71993-8997-468c-be2d-426b6b17d47c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:54:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg namespace: kserve-ci-e2e-test resourceVersion: "18228" uid: e737beeb-91f7-4517-9449-72b17679b380 kind: Event lastTimestamp: "2026-07-06T20:54:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T20:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:54:43Z" name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg.18bfcebbc1fea98a namespace: kserve-ci-e2e-test resourceVersion: "19518" uid: b1d0af85-5d16-4653-be4a-c52d34c97744 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:54:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg namespace: kserve-ci-e2e-test resourceVersion: "18228" uid: e737beeb-91f7-4517-9449-72b17679b380 kind: Event lastTimestamp: "2026-07-06T20:54:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:54:43Z" name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg.18bfcebbc1fef3d8 namespace: kserve-ci-e2e-test resourceVersion: "19519" uid: 77a98efb-5e7e-45fb-b48d-a7cf4fb57f0c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:54:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg namespace: kserve-ci-e2e-test resourceVersion: "18228" uid: e737beeb-91f7-4517-9449-72b17679b380 kind: Event lastTimestamp: "2026-07-06T20:54:52Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:54:52Z" name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg.18bfcebcc9a83b81 namespace: kserve-ci-e2e-test resourceVersion: "19637" uid: c9752b0e-652d-4049-a429-ceaaee204ac8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:51:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667 namespace: kserve-ci-e2e-test resourceVersion: "18203" uid: f50f3d37-f19c-45e8-99bd-aa24b25ac6b1 kind: Event lastTimestamp: "2026-07-06T20:51:53Z" message: 'Created pod: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667-b9zdg' metadata: creationTimestamp: "2026-07-06T20:51:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:51:53Z" name: isvc-sklearn-scale-raw-26dd6-predictor-7468f89667.18bfce942484d3cb namespace: kserve-ci-e2e-test resourceVersion: "18229" uid: 131d4e63-5678-47e2-9418-cdee305ff405 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:51:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-26dd6-predictor namespace: kserve-ci-e2e-test resourceVersion: "18201" uid: 79e5b9c9-6163-44e0-a635-07a23789131f kind: Event lastTimestamp: "2026-07-06T20:51:53Z" message: Scaled up replica set isvc-sklearn-scale-raw-26dd6-predictor-7468f89667 from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:51:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:51:53Z" name: isvc-sklearn-scale-raw-26dd6-predictor.18bfce941e67a0b6 namespace: kserve-ci-e2e-test resourceVersion: "18204" uid: f1e75915-e77c-4cea-a18b-f3c9a8a1ce1e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:52:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-26dd6-predictor namespace: kserve-ci-e2e-test resourceVersion: "18202" uid: 87cb9070-e378-43db-817f-d8d292a454d8 kind: Event lastTimestamp: "2026-07-06T20:52:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:52:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:52:23Z" name: isvc-sklearn-scale-raw-26dd6-predictor.18bfce979d7f1dfb namespace: kserve-ci-e2e-test resourceVersion: "18512" uid: c1715848-f430-43de-9025-9c588699907f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:52:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-26dd6-predictor namespace: kserve-ci-e2e-test resourceVersion: "18202" uid: 87cb9070-e378-43db-817f-d8d292a454d8 kind: Event lastTimestamp: "2026-07-06T20:52:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:52:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:52:23Z" name: isvc-sklearn-scale-raw-26dd6-predictor.18bfce979e083294 namespace: kserve-ci-e2e-test resourceVersion: "18513" uid: 23f0c95a-3241-41e1-a3d2-b73493661dcf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-06T20:52:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-26dd6-predictor namespace: kserve-ci-e2e-test resourceVersion: "18404" uid: 87cb9070-e378-43db-817f-d8d292a454d8 kind: Event lastTimestamp: "2026-07-06T20:54:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:52:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:54:38Z" name: isvc-sklearn-scale-raw-26dd6-predictor.18bfce9e9c920377 namespace: kserve-ci-e2e-test resourceVersion: "19461" uid: 9f437e86-be37-454b-a239-476358c86912 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-06T20:52:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-26dd6-predictor namespace: kserve-ci-e2e-test resourceVersion: "18404" uid: 87cb9070-e378-43db-817f-d8d292a454d8 kind: Event lastTimestamp: "2026-07-06T20:54:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:52:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:54:38Z" name: isvc-sklearn-scale-raw-26dd6-predictor.18bfce9e9d3650b8 namespace: kserve-ci-e2e-test resourceVersion: "19463" uid: 368dc0cb-d26d-409e-a839-5b1b06a45e2b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-06T20:51:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-26dd6 namespace: kserve-ci-e2e-test resourceVersion: "18192" uid: 955cb2ff-9c41-4935-a056-a908e6ff5c56 kind: Event lastTimestamp: "2026-07-06T20:54:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:51:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:54:33Z" name: isvc-sklearn-scale-raw-26dd6.18bfce941e7dffce namespace: kserve-ci-e2e-test resourceVersion: "19437" uid: 2ade5e97-5178-466d-b7c3-4e2c635611a1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:51:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-26dd6 namespace: kserve-ci-e2e-test resourceVersion: "18192" uid: 955cb2ff-9c41-4935-a056-a908e6ff5c56 kind: Event lastTimestamp: "2026-07-06T20:51:53Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-26dd6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-26dd6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:51:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:51:53Z" name: isvc-sklearn-scale-raw-26dd6.18bfce941f0ab2b6 namespace: kserve-ci-e2e-test resourceVersion: "18207" uid: db4e738a-e34d-4894-8ca5-c81bb7dd2385 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:54:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-26dd6 namespace: kserve-ci-e2e-test resourceVersion: "19435" uid: 955cb2ff-9c41-4935-a056-a908e6ff5c56 kind: Event lastTimestamp: "2026-07-06T20:54:33Z" message: InferenceService [isvc-sklearn-scale-raw-26dd6] is Ready metadata: creationTimestamp: "2026-07-06T20:54:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:54:33Z" name: isvc-sklearn-scale-raw-26dd6.18bfceb995ad484a namespace: kserve-ci-e2e-test resourceVersion: "19436" uid: 01a3bf45-68e2-4835-a8f1-cbf9ee6576f3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:46:26.967342Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs namespace: kserve-ci-e2e-test resourceVersion: "15246" uid: 92ec8a99-fa22-4b93-8856-017020efa067 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs to ip-10-0-139-250.ec2.internal metadata: creationTimestamp: "2026-07-06T20:46:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:46:26Z" name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs.18bfce4833ff761e namespace: kserve-ci-e2e-test resourceVersion: "15253" uid: f6e10e14-e377-4585-9403-d2a0892142b0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-95cfcc986-db5pj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:27Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: 92ec8a99-fa22-4b93-8856-017020efa067 kind: Event lastTimestamp: "2026-07-06T20:46:27Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:46:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:46:27Z" name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs.18bfce48528cc781 namespace: kserve-ci-e2e-test resourceVersion: "15279" uid: ec561b51-e00f-4210-8296-3a27b50c48ab reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: 92ec8a99-fa22-4b93-8856-017020efa067 kind: Event lastTimestamp: "2026-07-06T20:46:27Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:9b7f9014cc3727e4fd95a945572d3f70017252d596bd8941a1887e4a1514875c" already present on machine metadata: creationTimestamp: "2026-07-06T20:46:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:27Z" name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs.18bfce4854e48ca6 namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: 3909fcb5-cd2b-4fab-9f5d-7c3f42223aa2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: 92ec8a99-fa22-4b93-8856-017020efa067 kind: Event lastTimestamp: "2026-07-06T20:46:27Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-06T20:46:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:27Z" name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs.18bfce485748fb3c namespace: kserve-ci-e2e-test resourceVersion: "15284" uid: 5d258f8a-89bd-49dc-aa32-52374a85e13d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: 92ec8a99-fa22-4b93-8856-017020efa067 kind: Event lastTimestamp: "2026-07-06T20:46:27Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-06T20:46:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:27Z" name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs.18bfce4857a9fa4a namespace: kserve-ci-e2e-test resourceVersion: "15285" uid: 6f1c91c8-4e1d-4cf4-929b-ab5782572453 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: 92ec8a99-fa22-4b93-8856-017020efa067 kind: Event lastTimestamp: "2026-07-06T20:46:31Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-06T20:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:31Z" name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs.18bfce493affefde namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 32edda81-50ac-44ca-841b-2cd82e11283d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: 92ec8a99-fa22-4b93-8856-017020efa067 kind: Event lastTimestamp: "2026-07-06T20:46:48Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 17.481s (17.481s including waiting). Image size: 1305293053 bytes.' metadata: creationTimestamp: "2026-07-06T20:46:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:48Z" name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs.18bfce4d4cfee656 namespace: kserve-ci-e2e-test resourceVersion: "15479" uid: 67f99da9-8891-4352-9388-0cab4812b4af reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: 92ec8a99-fa22-4b93-8856-017020efa067 kind: Event lastTimestamp: "2026-07-06T20:46:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:46:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:48Z" name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs.18bfce4d5025d452 namespace: kserve-ci-e2e-test resourceVersion: "15481" uid: bfd7203c-09fe-4bd1-8c61-ffbf089470bc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: 92ec8a99-fa22-4b93-8856-017020efa067 kind: Event lastTimestamp: "2026-07-06T20:46:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:46:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:48Z" name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs.18bfce4d509c9769 namespace: kserve-ci-e2e-test resourceVersion: "15482" uid: 2b4eed39-cd75-43d1-a5bb-9b5c1dfdfa7a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: 92ec8a99-fa22-4b93-8856-017020efa067 kind: Event lastTimestamp: "2026-07-06T20:46:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T20:46:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:48Z" name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs.18bfce4d50a6641d namespace: kserve-ci-e2e-test resourceVersion: "15483" uid: 34b629d0-602d-444d-aaba-2fe055a762b5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: 92ec8a99-fa22-4b93-8856-017020efa067 kind: Event lastTimestamp: "2026-07-06T20:46:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:46:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:48Z" name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs.18bfce4d539a9465 namespace: kserve-ci-e2e-test resourceVersion: "15484" uid: 2c51be82-1f66-4dfe-88c0-8a1dd4fccc4e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: 92ec8a99-fa22-4b93-8856-017020efa067 kind: Event lastTimestamp: "2026-07-06T20:46:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:46:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:46:48Z" name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs.18bfce4d5426ad81 namespace: kserve-ci-e2e-test resourceVersion: "15485" uid: 2f671f8e-c72a-44d7-8423-15859896a611 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-06T20:46:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: 92ec8a99-fa22-4b93-8856-017020efa067 kind: Event lastTimestamp: "2026-07-06T20:47:45Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:47:45Z" name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs.18bfce4d716dadd4 namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 0b2194a8-cdfb-4e1b-98ee-4c4f6a223001 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: 92ec8a99-fa22-4b93-8856-017020efa067 kind: Event lastTimestamp: "2026-07-06T20:48:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T20:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:17Z" name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs.18bfce6200376c1d namespace: kserve-ci-e2e-test resourceVersion: "16306" uid: 0a539f79-ca9a-4b48-858b-8100a8e73efb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: 92ec8a99-fa22-4b93-8856-017020efa067 kind: Event lastTimestamp: "2026-07-06T20:48:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:17Z" name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs.18bfce62003a5e04 namespace: kserve-ci-e2e-test resourceVersion: "16311" uid: a63110c4-fc8a-42e2-a7e7-e7a2fb7ff586 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: 92ec8a99-fa22-4b93-8856-017020efa067 kind: Event lastTimestamp: "2026-07-06T20:48:20Z" message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp 10.134.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:48:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:20Z" name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs.18bfce62a1a6525e namespace: kserve-ci-e2e-test resourceVersion: "16360" uid: 19ff892a-51c4-473a-8fb1-fbaa6e3be3b1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f namespace: kserve-ci-e2e-test resourceVersion: "15230" uid: 2f1f539a-5dff-4f91-a5e2-451af354f3d9 kind: Event lastTimestamp: "2026-07-06T20:46:26Z" message: 'Created pod: isvc-xgboost-graph-raw-b4160-predictor-795b89786f-nlmbs' metadata: creationTimestamp: "2026-07-06T20:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:46:26Z" name: isvc-xgboost-graph-raw-b4160-predictor-795b89786f.18bfce48330a7bb5 namespace: kserve-ci-e2e-test resourceVersion: "15248" uid: ee2c53b6-ad53-4c41-b5c3-51db37d7cca0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-b4160-predictor namespace: kserve-ci-e2e-test resourceVersion: "15225" uid: 371a6e99-481e-4c6b-b512-2fcf97e50012 kind: Event lastTimestamp: "2026-07-06T20:46:26Z" message: Scaled up replica set isvc-xgboost-graph-raw-b4160-predictor-795b89786f from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:46:26Z" name: isvc-xgboost-graph-raw-b4160-predictor.18bfce482e9e5b45 namespace: kserve-ci-e2e-test resourceVersion: "15235" uid: a6304500-e960-432a-9083-c51f7c5e85e0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:46:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-b4160-predictor namespace: kserve-ci-e2e-test resourceVersion: "15228" uid: 66ccbd31-5afa-4b5b-855c-b007239f03e9 kind: Event lastTimestamp: "2026-07-06T20:47:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:46:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:47:26Z" name: isvc-xgboost-graph-raw-b4160-predictor.18bfce4bad360f46 namespace: kserve-ci-e2e-test resourceVersion: "15770" uid: 2f00339c-f400-4992-b0eb-da0900413261 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-06T20:46:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-b4160-predictor namespace: kserve-ci-e2e-test resourceVersion: "15228" uid: 66ccbd31-5afa-4b5b-855c-b007239f03e9 kind: Event lastTimestamp: "2026-07-06T20:47:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:46:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:47:26Z" name: isvc-xgboost-graph-raw-b4160-predictor.18bfce4badb7add1 namespace: kserve-ci-e2e-test resourceVersion: "15771" uid: 648cd8b6-0c5b-4319-8da7-abb87d50c49c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T20:47:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-b4160-predictor namespace: kserve-ci-e2e-test resourceVersion: "15430" uid: 66ccbd31-5afa-4b5b-855c-b007239f03e9 kind: Event lastTimestamp: "2026-07-06T20:48:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:47:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:11Z" name: isvc-xgboost-graph-raw-b4160-predictor.18bfce52ab58ba5c namespace: kserve-ci-e2e-test resourceVersion: "16141" uid: 99b8a043-96bb-4640-8537-2d0012f476a5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T20:47:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-b4160-predictor namespace: kserve-ci-e2e-test resourceVersion: "15430" uid: 66ccbd31-5afa-4b5b-855c-b007239f03e9 kind: Event lastTimestamp: "2026-07-06T20:48:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:47:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:11Z" name: isvc-xgboost-graph-raw-b4160-predictor.18bfce52abdaf367 namespace: kserve-ci-e2e-test resourceVersion: "16142" uid: d799a40f-95b5-4b15-b3fb-6cf8fe19b72a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-06T20:46:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-b4160 namespace: kserve-ci-e2e-test resourceVersion: "15197" uid: 87e7af23-98f9-4c4b-943e-f080d0b42169 kind: Event lastTimestamp: "2026-07-06T20:47:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:47:55Z" name: isvc-xgboost-graph-raw-b4160.18bfce482ec2d177 namespace: kserve-ci-e2e-test resourceVersion: "15993" uid: da51cd3a-2fc4-4138-87de-01b47ddbfbfc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:46:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-b4160 namespace: kserve-ci-e2e-test resourceVersion: "15197" uid: 87e7af23-98f9-4c4b-943e-f080d0b42169 kind: Event lastTimestamp: "2026-07-06T20:46:26Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-b4160": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-b4160": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:46:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:46:26Z" name: isvc-xgboost-graph-raw-b4160.18bfce482fd3f61e namespace: kserve-ci-e2e-test resourceVersion: "15239" uid: 8c732834-367d-4867-8530-f7782c50d305 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:47:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-b4160 namespace: kserve-ci-e2e-test resourceVersion: "15990" uid: 87e7af23-98f9-4c4b-943e-f080d0b42169 kind: Event lastTimestamp: "2026-07-06T20:47:55Z" message: InferenceService [isvc-xgboost-graph-raw-b4160] is Ready metadata: creationTimestamp: "2026-07-06T20:47:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:47:55Z" name: isvc-xgboost-graph-raw-b4160.18bfce5cd656c736 namespace: kserve-ci-e2e-test resourceVersion: "15991" uid: 1520411a-0fd3-4951-be00-c70b494b1381 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:48:17.598934Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: 834395f7-75f2-4a3c-a0fa-e8d3e1552e05 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx to ip-10-0-139-250.ec2.internal metadata: creationTimestamp: "2026-07-06T20:48:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:48:17Z" name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx.18bfce61f6279f49 namespace: kserve-ci-e2e-test resourceVersion: "16282" uid: 4f532820-3bef-4abf-a09e-9e626fef8449 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-95cfcc986-db5pj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:18Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx namespace: kserve-ci-e2e-test resourceVersion: "16286" uid: 834395f7-75f2-4a3c-a0fa-e8d3e1552e05 kind: Event lastTimestamp: "2026-07-06T20:48:18Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:48:18Z" name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx.18bfce620fd77519 namespace: kserve-ci-e2e-test resourceVersion: "16323" uid: 00203d76-b02a-4ebe-a496-c20becb237cd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: 834395f7-75f2-4a3c-a0fa-e8d3e1552e05 kind: Event lastTimestamp: "2026-07-06T20:48:18Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:9b7f9014cc3727e4fd95a945572d3f70017252d596bd8941a1887e4a1514875c" already present on machine metadata: creationTimestamp: "2026-07-06T20:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:18Z" name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx.18bfce62116ed0f5 namespace: kserve-ci-e2e-test resourceVersion: "16325" uid: 0f9af06a-af56-474e-adf5-ab34fe1f0df0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: 834395f7-75f2-4a3c-a0fa-e8d3e1552e05 kind: Event lastTimestamp: "2026-07-06T20:48:18Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-06T20:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:18Z" name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx.18bfce621447d15a namespace: kserve-ci-e2e-test resourceVersion: "16326" uid: 8ae030b9-c60a-4bdd-9662-626d9410e56f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: 834395f7-75f2-4a3c-a0fa-e8d3e1552e05 kind: Event lastTimestamp: "2026-07-06T20:48:18Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-06T20:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:18Z" name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx.18bfce6214d80588 namespace: kserve-ci-e2e-test resourceVersion: "16327" uid: dc9a0811-b728-46aa-8523-93190850b5ec reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: 834395f7-75f2-4a3c-a0fa-e8d3e1552e05 kind: Event lastTimestamp: "2026-07-06T20:48:21Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-07-06T20:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:21Z" name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx.18bfce62f243074f namespace: kserve-ci-e2e-test resourceVersion: "16372" uid: c041c666-8305-49cd-a74e-c43b8e6822b9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: 834395f7-75f2-4a3c-a0fa-e8d3e1552e05 kind: Event lastTimestamp: "2026-07-06T20:48:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:21Z" name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx.18bfce62f55424a5 namespace: kserve-ci-e2e-test resourceVersion: "16376" uid: 989c33d1-4820-4bcf-8dd5-250eebd1ab34 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: 834395f7-75f2-4a3c-a0fa-e8d3e1552e05 kind: Event lastTimestamp: "2026-07-06T20:48:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:21Z" name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx.18bfce62f5c7220e namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: 2e24f4ac-0b3c-4cf4-992b-8cfe1dc1962a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: 834395f7-75f2-4a3c-a0fa-e8d3e1552e05 kind: Event lastTimestamp: "2026-07-06T20:48:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T20:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:21Z" name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx.18bfce62f5d1cb61 namespace: kserve-ci-e2e-test resourceVersion: "16382" uid: ccff20f1-8a68-45a0-85b9-c9dd57973e1c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: 834395f7-75f2-4a3c-a0fa-e8d3e1552e05 kind: Event lastTimestamp: "2026-07-06T20:48:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:21Z" name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx.18bfce62f8685662 namespace: kserve-ci-e2e-test resourceVersion: "16388" uid: c2906143-67d0-461c-aed6-4dc71f3d48ae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: 834395f7-75f2-4a3c-a0fa-e8d3e1552e05 kind: Event lastTimestamp: "2026-07-06T20:48:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:21Z" name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx.18bfce62f8d1a349 namespace: kserve-ci-e2e-test resourceVersion: "16390" uid: 50bbf7e3-32e4-403c-bf34-964d12be1d58 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-06T20:48:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: 834395f7-75f2-4a3c-a0fa-e8d3e1552e05 kind: Event lastTimestamp: "2026-07-06T20:50:09Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:48:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:09Z" name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx.18bfce636a7a52e9 namespace: kserve-ci-e2e-test resourceVersion: "17350" uid: 34e9054e-9dbf-478d-8b09-debc14259b6b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: 834395f7-75f2-4a3c-a0fa-e8d3e1552e05 kind: Event lastTimestamp: "2026-07-06T20:50:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T20:50:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:07Z" name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx.18bfce7ba32a583c namespace: kserve-ci-e2e-test resourceVersion: "17318" uid: 4f737631-64f1-4ccc-a9ef-50ee92e7baa8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: 834395f7-75f2-4a3c-a0fa-e8d3e1552e05 kind: Event lastTimestamp: "2026-07-06T20:50:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:50:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:07Z" name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx.18bfce7ba32ad162 namespace: kserve-ci-e2e-test resourceVersion: "17321" uid: e7fd2a1f-a9e7-435a-9680-fe58b09a0752 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: 834395f7-75f2-4a3c-a0fa-e8d3e1552e05 kind: Event lastTimestamp: "2026-07-06T20:50:09Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:50:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:09Z" name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx.18bfce7c18cfb693 namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: 048c5334-a766-496e-9520-1f55823a89ad reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5 namespace: kserve-ci-e2e-test resourceVersion: "16245" uid: 94c3f0fc-bbbb-4c40-b1ad-a326da455464 kind: Event lastTimestamp: "2026-07-06T20:48:17Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5-x6hzx' metadata: creationTimestamp: "2026-07-06T20:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:17Z" name: isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5.18bfce61f575430f namespace: kserve-ci-e2e-test resourceVersion: "16280" uid: e3a74745-a0fc-45df-b581-d123cf769421 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-88d19-predictor namespace: kserve-ci-e2e-test resourceVersion: "16238" uid: 34ad562c-41bb-471a-878e-5715b577372d kind: Event lastTimestamp: "2026-07-06T20:48:17Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-88d19-predictor-7b9b479bb5 from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:17Z" name: isvc-xgboost-graph-raw-hpa-88d19-predictor.18bfce61ebf4d148 namespace: kserve-ci-e2e-test resourceVersion: "16251" uid: b8869e5f-1dce-4de3-a062-a1de7f0c8440 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:48:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-88d19-predictor namespace: kserve-ci-e2e-test resourceVersion: "16247" uid: 34f3d544-8a08-4b27-8f58-9179b5a7c482 kind: Event lastTimestamp: "2026-07-06T20:49:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:48:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:49:02Z" name: isvc-xgboost-graph-raw-hpa-88d19-predictor.18bfce656ae12af9 namespace: kserve-ci-e2e-test resourceVersion: "16706" uid: 34a1a5da-fb4b-4109-8ede-9eec9ac657a8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:48:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-88d19-predictor namespace: kserve-ci-e2e-test resourceVersion: "16247" uid: 34f3d544-8a08-4b27-8f58-9179b5a7c482 kind: Event lastTimestamp: "2026-07-06T20:49:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:48:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:49:02Z" name: isvc-xgboost-graph-raw-hpa-88d19-predictor.18bfce656b881a26 namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: 2b91fb1f-1b38-4305-8055-93a487575f10 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T20:48:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-88d19-predictor namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 34f3d544-8a08-4b27-8f58-9179b5a7c482 kind: Event lastTimestamp: "2026-07-06T20:49:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:48:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:49:47Z" name: isvc-xgboost-graph-raw-hpa-88d19-predictor.18bfce68ebc31747 namespace: kserve-ci-e2e-test resourceVersion: "17053" uid: 70607815-a45b-499a-8cda-33e4df796d6c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T20:48:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-88d19-predictor namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 34f3d544-8a08-4b27-8f58-9179b5a7c482 kind: Event lastTimestamp: "2026-07-06T20:49:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:48:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:49:47Z" name: isvc-xgboost-graph-raw-hpa-88d19-predictor.18bfce68ec9ff971 namespace: kserve-ci-e2e-test resourceVersion: "17054" uid: fc669f49-613f-4889-b745-b44dbabb67cb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-06T20:48:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-88d19 namespace: kserve-ci-e2e-test resourceVersion: "16227" uid: 738c63e2-7655-413c-9a11-d69e87ea9515 kind: Event lastTimestamp: "2026-07-06T20:49:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:49:20Z" name: isvc-xgboost-graph-raw-hpa-88d19.18bfce61eca59241 namespace: kserve-ci-e2e-test resourceVersion: "16844" uid: 44683185-6a64-4c3b-a52a-a45b4056a29f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-88d19 namespace: kserve-ci-e2e-test resourceVersion: "16227" uid: 738c63e2-7655-413c-9a11-d69e87ea9515 kind: Event lastTimestamp: "2026-07-06T20:48:17Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-hpa-88d19": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-hpa-88d19": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:48:17Z" name: isvc-xgboost-graph-raw-hpa-88d19.18bfce61ef45e864 namespace: kserve-ci-e2e-test resourceVersion: "16265" uid: e7c5a809-ff0c-45bb-92fe-be43ea506535 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:49:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-88d19 namespace: kserve-ci-e2e-test resourceVersion: "16841" uid: 738c63e2-7655-413c-9a11-d69e87ea9515 kind: Event lastTimestamp: "2026-07-06T20:49:19Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-88d19] is Ready metadata: creationTimestamp: "2026-07-06T20:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:49:19Z" name: isvc-xgboost-graph-raw-hpa-88d19.18bfce707b5eff3d namespace: kserve-ci-e2e-test resourceVersion: "16842" uid: bb194b6e-930c-4ee1-96c5-091b2347d25a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:50:07.741050Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-e7cb2-predictor-689764df45-x5zcv namespace: kserve-ci-e2e-test resourceVersion: "17276" uid: 1bdbbb2c-7909-4561-9234-211d7d74223b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-e7cb2-predictor-689764df45-x5zcv to ip-10-0-139-250.ec2.internal metadata: creationTimestamp: "2026-07-06T20:50:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:50:07Z" name: message-dumper-raw-e7cb2-predictor-689764df45-x5zcv.18bfce7b9b22db0b namespace: kserve-ci-e2e-test resourceVersion: "17284" uid: 72bf897e-a278-41e4-89ad-ff3207c43e5d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-95cfcc986-db5pj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:08Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-e7cb2-predictor-689764df45-x5zcv namespace: kserve-ci-e2e-test resourceVersion: "17293" uid: 1bdbbb2c-7909-4561-9234-211d7d74223b kind: Event lastTimestamp: "2026-07-06T20:50:08Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:50:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:50:08Z" name: message-dumper-raw-e7cb2-predictor-689764df45-x5zcv.18bfce7bb4cfa096 namespace: kserve-ci-e2e-test resourceVersion: "17328" uid: 23416145-4c97-4f8e-a0c7-36643445839c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-e7cb2-predictor-689764df45-x5zcv namespace: kserve-ci-e2e-test resourceVersion: "17279" uid: 1bdbbb2c-7909-4561-9234-211d7d74223b kind: Event lastTimestamp: "2026-07-06T20:50:08Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-06T20:50:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:08Z" name: message-dumper-raw-e7cb2-predictor-689764df45-x5zcv.18bfce7bb666e2ab namespace: kserve-ci-e2e-test resourceVersion: "17330" uid: cd4a4209-1dbc-4232-859e-26dbd92b9eda reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-e7cb2-predictor-689764df45-x5zcv namespace: kserve-ci-e2e-test resourceVersion: "17279" uid: 1bdbbb2c-7909-4561-9234-211d7d74223b kind: Event lastTimestamp: "2026-07-06T20:50:09Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.325s (1.325s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-06T20:50:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:09Z" name: message-dumper-raw-e7cb2-predictor-689764df45-x5zcv.18bfce7c0567f456 namespace: kserve-ci-e2e-test resourceVersion: "17339" uid: 46d8666a-5bf6-4d4d-af34-3f5ee8d01243 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-e7cb2-predictor-689764df45-x5zcv namespace: kserve-ci-e2e-test resourceVersion: "17279" uid: 1bdbbb2c-7909-4561-9234-211d7d74223b kind: Event lastTimestamp: "2026-07-06T20:50:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:50:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:09Z" name: message-dumper-raw-e7cb2-predictor-689764df45-x5zcv.18bfce7c085ba312 namespace: kserve-ci-e2e-test resourceVersion: "17340" uid: b3a9f462-f839-465b-9c26-29f6d51da00a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-e7cb2-predictor-689764df45-x5zcv namespace: kserve-ci-e2e-test resourceVersion: "17279" uid: 1bdbbb2c-7909-4561-9234-211d7d74223b kind: Event lastTimestamp: "2026-07-06T20:50:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:50:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:09Z" name: message-dumper-raw-e7cb2-predictor-689764df45-x5zcv.18bfce7c08cd1c39 namespace: kserve-ci-e2e-test resourceVersion: "17341" uid: 5f918b72-2946-4c03-9fc4-cef661c993c7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-e7cb2-predictor-689764df45-x5zcv namespace: kserve-ci-e2e-test resourceVersion: "17279" uid: 1bdbbb2c-7909-4561-9234-211d7d74223b kind: Event lastTimestamp: "2026-07-06T20:50:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T20:50:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:09Z" name: message-dumper-raw-e7cb2-predictor-689764df45-x5zcv.18bfce7c08d76599 namespace: kserve-ci-e2e-test resourceVersion: "17342" uid: 0105c88c-8a3e-47ea-b8f6-8344b0bdc2e3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-e7cb2-predictor-689764df45-x5zcv namespace: kserve-ci-e2e-test resourceVersion: "17279" uid: 1bdbbb2c-7909-4561-9234-211d7d74223b kind: Event lastTimestamp: "2026-07-06T20:50:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:50:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:09Z" name: message-dumper-raw-e7cb2-predictor-689764df45-x5zcv.18bfce7c0cc93a44 namespace: kserve-ci-e2e-test resourceVersion: "17344" uid: 9eb50bda-04ed-411b-8851-1ec6c18a7793 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-e7cb2-predictor-689764df45-x5zcv namespace: kserve-ci-e2e-test resourceVersion: "17279" uid: 1bdbbb2c-7909-4561-9234-211d7d74223b kind: Event lastTimestamp: "2026-07-06T20:50:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:50:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:09Z" name: message-dumper-raw-e7cb2-predictor-689764df45-x5zcv.18bfce7c0d442abc namespace: kserve-ci-e2e-test resourceVersion: "17345" uid: 900eaa72-7cd3-42f2-97f0-41204dce255e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:51:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-e7cb2-predictor-689764df45-x5zcv namespace: kserve-ci-e2e-test resourceVersion: "17279" uid: 1bdbbb2c-7909-4561-9234-211d7d74223b kind: Event lastTimestamp: "2026-07-06T20:51:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T20:51:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:51:53Z" name: message-dumper-raw-e7cb2-predictor-689764df45-x5zcv.18bfce9428ba07f9 namespace: kserve-ci-e2e-test resourceVersion: "18257" uid: 7c6cfd3c-f253-4fbb-af82-6745f515fddf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:51:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-e7cb2-predictor-689764df45-x5zcv namespace: kserve-ci-e2e-test resourceVersion: "17279" uid: 1bdbbb2c-7909-4561-9234-211d7d74223b kind: Event lastTimestamp: "2026-07-06T20:51:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:51:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:51:53Z" name: message-dumper-raw-e7cb2-predictor-689764df45-x5zcv.18bfce9428bb5a21 namespace: kserve-ci-e2e-test resourceVersion: "18259" uid: 2b2904ee-32db-47bd-b626-ca6206f584c2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-e7cb2-predictor-689764df45 namespace: kserve-ci-e2e-test resourceVersion: "17263" uid: 5bcac498-f0c0-41bb-80a9-f01f67412839 kind: Event lastTimestamp: "2026-07-06T20:50:07Z" message: 'Created pod: message-dumper-raw-e7cb2-predictor-689764df45-x5zcv' metadata: creationTimestamp: "2026-07-06T20:50:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:50:07Z" name: message-dumper-raw-e7cb2-predictor-689764df45.18bfce7b9a919e18 namespace: kserve-ci-e2e-test resourceVersion: "17280" uid: 178ec1fd-3d83-4d72-b2f2-58ff89a813a2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-e7cb2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17261" uid: 193ca1dc-6871-4033-b642-3f4e28a04f37 kind: Event lastTimestamp: "2026-07-06T20:50:07Z" message: Scaled up replica set message-dumper-raw-e7cb2-predictor-689764df45 from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:50:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:50:07Z" name: message-dumper-raw-e7cb2-predictor.18bfce7b966664d2 namespace: kserve-ci-e2e-test resourceVersion: "17264" uid: 61504198-7468-4c87-8411-73da99997909 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:50:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-e7cb2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17262" uid: 0ae52230-9a48-406d-b3b3-8bb4d7fd681d kind: Event lastTimestamp: "2026-07-06T20:50:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:50:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:50:37Z" name: message-dumper-raw-e7cb2-predictor.18bfce7f14f02816 namespace: kserve-ci-e2e-test resourceVersion: "17643" uid: 41b5403a-fe2a-4d7c-bd62-231a4aa2ec30 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:50:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-e7cb2-predictor namespace: kserve-ci-e2e-test resourceVersion: "17262" uid: 0ae52230-9a48-406d-b3b3-8bb4d7fd681d kind: Event lastTimestamp: "2026-07-06T20:50:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:50:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:50:37Z" name: message-dumper-raw-e7cb2-predictor.18bfce7f15841c31 namespace: kserve-ci-e2e-test resourceVersion: "17644" uid: a0e36aa1-034d-4251-9435-8b04a229c5f8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-06T20:50:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-e7cb2 namespace: kserve-ci-e2e-test resourceVersion: "17252" uid: 52ece6bf-251e-43dd-87d3-dd0155df5359 kind: Event lastTimestamp: "2026-07-06T20:50:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:50:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:50:17Z" name: message-dumper-raw-e7cb2.18bfce7b96691284 namespace: kserve-ci-e2e-test resourceVersion: "17417" uid: bcdc1cc5-1430-445f-8d2e-e5672397c8ae reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-e7cb2 namespace: kserve-ci-e2e-test resourceVersion: "17252" uid: 52ece6bf-251e-43dd-87d3-dd0155df5359 kind: Event lastTimestamp: "2026-07-06T20:50:07Z" message: 'Failed to update status for InferenceService "message-dumper-raw-e7cb2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-e7cb2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:50:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:50:07Z" name: message-dumper-raw-e7cb2.18bfce7b9795cf3c namespace: kserve-ci-e2e-test resourceVersion: "17267" uid: b248151e-9061-4a5d-a0b6-31b63bbafc76 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-e7cb2 namespace: kserve-ci-e2e-test resourceVersion: "17413" uid: 52ece6bf-251e-43dd-87d3-dd0155df5359 kind: Event lastTimestamp: "2026-07-06T20:50:17Z" message: InferenceService [message-dumper-raw-e7cb2] is Ready metadata: creationTimestamp: "2026-07-06T20:50:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:50:17Z" name: message-dumper-raw-e7cb2.18bfce7ddc265570 namespace: kserve-ci-e2e-test resourceVersion: "17415" uid: 3ddfc993-8d36-419d-a924-5227f68f5700 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:48:06.964159Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-b4160-658c8f4f6-gtscw namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: f97eeaa6-2b5f-4be5-ac73-f4b28d0a4f3d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-b4160-658c8f4f6-gtscw to ip-10-0-129-46.ec2.internal metadata: creationTimestamp: "2026-07-06T20:48:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:48:06Z" name: model-chainer-raw-b4160-658c8f4f6-gtscw.18bfce5f7c45bbf9 namespace: kserve-ci-e2e-test resourceVersion: "16082" uid: 6d90cce4-6f63-4990-810d-645b4be3cae3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-95cfcc986-db5pj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:07Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-b4160-658c8f4f6-gtscw namespace: kserve-ci-e2e-test resourceVersion: "16085" uid: f97eeaa6-2b5f-4be5-ac73-f4b28d0a4f3d kind: Event lastTimestamp: "2026-07-06T20:48:07Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:48:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:48:07Z" name: model-chainer-raw-b4160-658c8f4f6-gtscw.18bfce5f962b510b namespace: kserve-ci-e2e-test resourceVersion: "16096" uid: 438773ad-0383-4ae2-947c-66610d3cbfdf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-b4160} kind: Pod name: model-chainer-raw-b4160-658c8f4f6-gtscw namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: f97eeaa6-2b5f-4be5-ac73-f4b28d0a4f3d kind: Event lastTimestamp: "2026-07-06T20:48:07Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:a8d64b572a976cc85b28488a3ef8f7d50ce28ddb75186c1859612e88d6dba080" metadata: creationTimestamp: "2026-07-06T20:48:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:07Z" name: model-chainer-raw-b4160-658c8f4f6-gtscw.18bfce5f97ac33ba namespace: kserve-ci-e2e-test resourceVersion: "16098" uid: 0ecff353-5132-4993-ab7a-5e5301e0c31f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-46.ec2.internal source: component: kubelet host: ip-10-0-129-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-b4160} kind: Pod name: model-chainer-raw-b4160-658c8f4f6-gtscw namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: f97eeaa6-2b5f-4be5-ac73-f4b28d0a4f3d kind: Event lastTimestamp: "2026-07-06T20:48:09Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:a8d64b572a976cc85b28488a3ef8f7d50ce28ddb75186c1859612e88d6dba080" in 2.057s (2.057s including waiting). Image size: 218071146 bytes.' metadata: creationTimestamp: "2026-07-06T20:48:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:09Z" name: model-chainer-raw-b4160-658c8f4f6-gtscw.18bfce6012526afa namespace: kserve-ci-e2e-test resourceVersion: "16113" uid: 6bebc53c-c965-4404-bf6d-c84a6174ce23 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-46.ec2.internal source: component: kubelet host: ip-10-0-129-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-b4160} kind: Pod name: model-chainer-raw-b4160-658c8f4f6-gtscw namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: f97eeaa6-2b5f-4be5-ac73-f4b28d0a4f3d kind: Event lastTimestamp: "2026-07-06T20:48:09Z" message: 'Created container: model-chainer-raw-b4160' metadata: creationTimestamp: "2026-07-06T20:48:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:09Z" name: model-chainer-raw-b4160-658c8f4f6-gtscw.18bfce60154ba974 namespace: kserve-ci-e2e-test resourceVersion: "16115" uid: 5814b85c-9506-4782-a170-e3731314f646 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-46.ec2.internal source: component: kubelet host: ip-10-0-129-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-b4160} kind: Pod name: model-chainer-raw-b4160-658c8f4f6-gtscw namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: f97eeaa6-2b5f-4be5-ac73-f4b28d0a4f3d kind: Event lastTimestamp: "2026-07-06T20:48:09Z" message: Started container model-chainer-raw-b4160 metadata: creationTimestamp: "2026-07-06T20:48:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:09Z" name: model-chainer-raw-b4160-658c8f4f6-gtscw.18bfce6015c8f5f9 namespace: kserve-ci-e2e-test resourceVersion: "16116" uid: 5de0eb46-4835-4635-ae5a-5c5105ab1c7e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-46.ec2.internal source: component: kubelet host: ip-10-0-129-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-b4160} kind: Pod name: model-chainer-raw-b4160-658c8f4f6-gtscw namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: f97eeaa6-2b5f-4be5-ac73-f4b28d0a4f3d kind: Event lastTimestamp: "2026-07-06T20:48:16Z" message: Stopping container model-chainer-raw-b4160 metadata: creationTimestamp: "2026-07-06T20:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:16Z" name: model-chainer-raw-b4160-658c8f4f6-gtscw.18bfce61cfc09354 namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: b7720a8d-4802-4c5c-831a-36dd6d94b079 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-46.ec2.internal source: component: kubelet host: ip-10-0-129-46.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T20:48:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-b4160} kind: Pod name: model-chainer-raw-b4160-658c8f4f6-gtscw namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: f97eeaa6-2b5f-4be5-ac73-f4b28d0a4f3d kind: Event lastTimestamp: "2026-07-06T20:48:45Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-06T20:48:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:48:45Z" name: model-chainer-raw-b4160-658c8f4f6-gtscw.18bfce62be673a31 namespace: kserve-ci-e2e-test resourceVersion: "16583" uid: 949c4102-f13b-4950-bec6-e55bda5e3b9d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-46.ec2.internal source: component: kubelet host: ip-10-0-129-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-b4160-658c8f4f6 namespace: kserve-ci-e2e-test resourceVersion: "16073" uid: a42f5bf7-2c97-4288-bc4a-e206e13b9ebf kind: Event lastTimestamp: "2026-07-06T20:48:06Z" message: 'Created pod: model-chainer-raw-b4160-658c8f4f6-gtscw' metadata: creationTimestamp: "2026-07-06T20:48:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:06Z" name: model-chainer-raw-b4160-658c8f4f6.18bfce5f7b545336 namespace: kserve-ci-e2e-test resourceVersion: "16079" uid: aa8713bc-ad0a-43af-adc0-b75db00bd74b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-b4160 namespace: kserve-ci-e2e-test resourceVersion: "16071" uid: 3b0a3e58-5a28-4f50-b62b-391eda5cfc8b kind: Event lastTimestamp: "2026-07-06T20:48:06Z" message: Scaled up replica set model-chainer-raw-b4160-658c8f4f6 from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:48:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:48:06Z" name: model-chainer-raw-b4160.18bfce5f7866ab68 namespace: kserve-ci-e2e-test resourceVersion: "16075" uid: 2f84f7a2-7f9a-4dff-aa6b-3bcc34e4197e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:48:07Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-b4160 namespace: kserve-ci-e2e-test resourceVersion: "16089" uid: 8e24ea1b-2352-45c9-befb-ba0d58962f8e kind: Event lastTimestamp: "2026-07-06T20:48:07Z" message: InferenceGraph [model-chainer-raw-b4160] is Ready metadata: creationTimestamp: "2026-07-06T20:48:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:48:07Z" name: model-chainer-raw-b4160.18bfce5f832dd9d4 namespace: kserve-ci-e2e-test resourceVersion: "16091" uid: 11c7b7b9-b8d1-4223-b44a-768b9e7f27f7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:49:57.322917Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-88d19-66946dd7d5-47rc7 namespace: kserve-ci-e2e-test resourceVersion: "17131" uid: 40bf2a87-20f4-4172-94d8-d17541d6fafb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-88d19-66946dd7d5-47rc7 to ip-10-0-129-46.ec2.internal metadata: creationTimestamp: "2026-07-06T20:49:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:49:57Z" name: model-chainer-raw-hpa-88d19-66946dd7d5-47rc7.18bfce792e2ac8d1 namespace: kserve-ci-e2e-test resourceVersion: "17136" uid: a57f1733-1c20-429a-a422-7db787e328c8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-95cfcc986-db5pj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:49:57Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-88d19-66946dd7d5-47rc7 namespace: kserve-ci-e2e-test resourceVersion: "17133" uid: 40bf2a87-20f4-4172-94d8-d17541d6fafb kind: Event lastTimestamp: "2026-07-06T20:49:57Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-hpa-88d19-serving-cert" not found' metadata: creationTimestamp: "2026-07-06T20:49:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:49:57Z" name: model-chainer-raw-hpa-88d19-66946dd7d5-47rc7.18bfce793493a20d namespace: kserve-ci-e2e-test resourceVersion: "17145" uid: eaccaaa4-8bc5-4ed2-ae63-cba8c09dfa06 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-46.ec2.internal source: component: kubelet host: ip-10-0-129-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:49:58Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-88d19-66946dd7d5-47rc7 namespace: kserve-ci-e2e-test resourceVersion: "17140" uid: 40bf2a87-20f4-4172-94d8-d17541d6fafb kind: Event lastTimestamp: "2026-07-06T20:49:58Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:49:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:49:58Z" name: model-chainer-raw-hpa-88d19-66946dd7d5-47rc7.18bfce7959e90c23 namespace: kserve-ci-e2e-test resourceVersion: "17155" uid: aee8f05c-d0b9-422d-94c3-e02f6555cad3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:49:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-88d19} kind: Pod name: model-chainer-raw-hpa-88d19-66946dd7d5-47rc7 namespace: kserve-ci-e2e-test resourceVersion: "17133" uid: 40bf2a87-20f4-4172-94d8-d17541d6fafb kind: Event lastTimestamp: "2026-07-06T20:49:58Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a8d64b572a976cc85b28488a3ef8f7d50ce28ddb75186c1859612e88d6dba080" already present on machine metadata: creationTimestamp: "2026-07-06T20:49:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:49:58Z" name: model-chainer-raw-hpa-88d19-66946dd7d5-47rc7.18bfce795b7103a0 namespace: kserve-ci-e2e-test resourceVersion: "17157" uid: 8f1b9454-92e5-40d3-b700-4fa940c8ae9b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-46.ec2.internal source: component: kubelet host: ip-10-0-129-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:49:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-88d19} kind: Pod name: model-chainer-raw-hpa-88d19-66946dd7d5-47rc7 namespace: kserve-ci-e2e-test resourceVersion: "17133" uid: 40bf2a87-20f4-4172-94d8-d17541d6fafb kind: Event lastTimestamp: "2026-07-06T20:49:58Z" message: 'Created container: model-chainer-raw-hpa-88d19' metadata: creationTimestamp: "2026-07-06T20:49:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:49:58Z" name: model-chainer-raw-hpa-88d19-66946dd7d5-47rc7.18bfce795e8c8786 namespace: kserve-ci-e2e-test resourceVersion: "17159" uid: 5d615e42-2f82-4011-90fb-22c3a96b851d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-46.ec2.internal source: component: kubelet host: ip-10-0-129-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:49:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-88d19} kind: Pod name: model-chainer-raw-hpa-88d19-66946dd7d5-47rc7 namespace: kserve-ci-e2e-test resourceVersion: "17133" uid: 40bf2a87-20f4-4172-94d8-d17541d6fafb kind: Event lastTimestamp: "2026-07-06T20:49:58Z" message: Started container model-chainer-raw-hpa-88d19 metadata: creationTimestamp: "2026-07-06T20:49:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:49:58Z" name: model-chainer-raw-hpa-88d19-66946dd7d5-47rc7.18bfce795f0505e1 namespace: kserve-ci-e2e-test resourceVersion: "17160" uid: ea0c079e-5745-41ad-89dd-e3f7a096e435 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-46.ec2.internal source: component: kubelet host: ip-10-0-129-46.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:50:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-88d19} kind: Pod name: model-chainer-raw-hpa-88d19-66946dd7d5-47rc7 namespace: kserve-ci-e2e-test resourceVersion: "17133" uid: 40bf2a87-20f4-4172-94d8-d17541d6fafb kind: Event lastTimestamp: "2026-07-06T20:50:07Z" message: Stopping container model-chainer-raw-hpa-88d19 metadata: creationTimestamp: "2026-07-06T20:50:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:07Z" name: model-chainer-raw-hpa-88d19-66946dd7d5-47rc7.18bfce7b84ea4e1e namespace: kserve-ci-e2e-test resourceVersion: "17246" uid: c7af69df-d328-4269-9fd6-3a7149f4dadf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-46.ec2.internal source: component: kubelet host: ip-10-0-129-46.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-06T20:50:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-88d19} kind: Pod name: model-chainer-raw-hpa-88d19-66946dd7d5-47rc7 namespace: kserve-ci-e2e-test resourceVersion: "17133" uid: 40bf2a87-20f4-4172-94d8-d17541d6fafb kind: Event lastTimestamp: "2026-07-06T20:50:34Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-06T20:50:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:50:34Z" name: model-chainer-raw-hpa-88d19-66946dd7d5-47rc7.18bfce7bff67e10e namespace: kserve-ci-e2e-test resourceVersion: "17623" uid: 4c96a627-eefa-497f-bb01-5220ee3ee2f3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-46.ec2.internal source: component: kubelet host: ip-10-0-129-46.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:49:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-88d19-66946dd7d5 namespace: kserve-ci-e2e-test resourceVersion: "17127" uid: 309ecfb4-0af8-4d21-9fcd-d78f07fb9639 kind: Event lastTimestamp: "2026-07-06T20:49:57Z" message: 'Created pod: model-chainer-raw-hpa-88d19-66946dd7d5-47rc7' metadata: creationTimestamp: "2026-07-06T20:49:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:49:57Z" name: model-chainer-raw-hpa-88d19-66946dd7d5.18bfce792d4d3e61 namespace: kserve-ci-e2e-test resourceVersion: "17134" uid: 2c0b7f5f-07ec-4cba-91ed-152cf7ff2a68 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:49:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-88d19 namespace: kserve-ci-e2e-test resourceVersion: "17124" uid: fbb815a7-cb76-42a4-be60-3178f00863d0 kind: Event lastTimestamp: "2026-07-06T20:49:57Z" message: Scaled up replica set model-chainer-raw-hpa-88d19-66946dd7d5 from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:49:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:49:57Z" name: model-chainer-raw-hpa-88d19.18bfce792ba1f0ec namespace: kserve-ci-e2e-test resourceVersion: "17128" uid: f3db438f-0f26-42e1-9517-b185a9f980dc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:49:57Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-88d19 namespace: kserve-ci-e2e-test resourceVersion: "17141" uid: ffdf5bef-ffc9-46fc-a142-8fbf2db5a8de kind: Event lastTimestamp: "2026-07-06T20:49:57Z" message: InferenceGraph [model-chainer-raw-hpa-88d19] is Ready metadata: creationTimestamp: "2026-07-06T20:49:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:49:57Z" name: model-chainer-raw-hpa-88d19.18bfce793043c105 namespace: kserve-ci-e2e-test resourceVersion: "17143" uid: ce8bc6f7-acca-4378-898f-cba5be125fd4 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:56:53.044170Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-ef99f-predictor-77b85567df-trpsw namespace: kserve-ci-e2e-test resourceVersion: "20710" uid: 84831045-9e8c-4f6f-9e53-2fbb1b4739e6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-ef99f-predictor-77b85567df-trpsw to ip-10-0-139-250.ec2.internal metadata: creationTimestamp: "2026-07-06T20:56:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:56:53Z" name: raw-sklearn-ef99f-predictor-77b85567df-trpsw.18bfced9f915afbd namespace: kserve-ci-e2e-test resourceVersion: "20716" uid: 316f5b6e-9765-4457-b25f-a975533b6fed reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-95cfcc986-db5pj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:53Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-ef99f-predictor-77b85567df-trpsw namespace: kserve-ci-e2e-test resourceVersion: "20721" uid: 84831045-9e8c-4f6f-9e53-2fbb1b4739e6 kind: Event lastTimestamp: "2026-07-06T20:56:53Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:56:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:56:53Z" name: raw-sklearn-ef99f-predictor-77b85567df-trpsw.18bfceda12c2a49b namespace: kserve-ci-e2e-test resourceVersion: "20729" uid: 432154b5-0097-418c-9c46-8c56e3de1898 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-ef99f-predictor-77b85567df-trpsw namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: 84831045-9e8c-4f6f-9e53-2fbb1b4739e6 kind: Event lastTimestamp: "2026-07-06T20:56:53Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:9b7f9014cc3727e4fd95a945572d3f70017252d596bd8941a1887e4a1514875c" already present on machine metadata: creationTimestamp: "2026-07-06T20:56:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:53Z" name: raw-sklearn-ef99f-predictor-77b85567df-trpsw.18bfceda1459dba3 namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: 1dc6b26c-82ae-4878-b2c0-de5e3848f630 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-ef99f-predictor-77b85567df-trpsw namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: 84831045-9e8c-4f6f-9e53-2fbb1b4739e6 kind: Event lastTimestamp: "2026-07-06T20:56:53Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-06T20:56:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:53Z" name: raw-sklearn-ef99f-predictor-77b85567df-trpsw.18bfceda16ecba53 namespace: kserve-ci-e2e-test resourceVersion: "20733" uid: 143d5ae2-0cef-4dd3-9835-40e6cffc353a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-ef99f-predictor-77b85567df-trpsw namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: 84831045-9e8c-4f6f-9e53-2fbb1b4739e6 kind: Event lastTimestamp: "2026-07-06T20:56:53Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-06T20:56:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:53Z" name: raw-sklearn-ef99f-predictor-77b85567df-trpsw.18bfceda1766b0ae namespace: kserve-ci-e2e-test resourceVersion: "20734" uid: 8471f197-9d8b-4bfb-a9e3-cca03652943b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-ef99f-predictor-77b85567df-trpsw namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: 84831045-9e8c-4f6f-9e53-2fbb1b4739e6 kind: Event lastTimestamp: "2026-07-06T20:56:57Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1703" already present on machine metadata: creationTimestamp: "2026-07-06T20:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:57Z" name: raw-sklearn-ef99f-predictor-77b85567df-trpsw.18bfcedb14e72c5a namespace: kserve-ci-e2e-test resourceVersion: "20772" uid: 51c61a6d-4823-40c8-b7b0-145384cf46b4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-ef99f-predictor-77b85567df-trpsw namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: 84831045-9e8c-4f6f-9e53-2fbb1b4739e6 kind: Event lastTimestamp: "2026-07-06T20:56:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:57Z" name: raw-sklearn-ef99f-predictor-77b85567df-trpsw.18bfcedb18854d70 namespace: kserve-ci-e2e-test resourceVersion: "20774" uid: 52a07591-08e3-4837-9e93-0271275740aa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-ef99f-predictor-77b85567df-trpsw namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: 84831045-9e8c-4f6f-9e53-2fbb1b4739e6 kind: Event lastTimestamp: "2026-07-06T20:56:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:57Z" name: raw-sklearn-ef99f-predictor-77b85567df-trpsw.18bfcedb18fff051 namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: 2f4c2f33-52b9-4085-849a-ca82a4d65522 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-ef99f-predictor-77b85567df-trpsw namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: 84831045-9e8c-4f6f-9e53-2fbb1b4739e6 kind: Event lastTimestamp: "2026-07-06T20:56:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T20:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:57Z" name: raw-sklearn-ef99f-predictor-77b85567df-trpsw.18bfcedb190c9ad0 namespace: kserve-ci-e2e-test resourceVersion: "20777" uid: 04bcd0b8-c393-42dd-bad9-a3e8378802f1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-ef99f-predictor-77b85567df-trpsw namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: 84831045-9e8c-4f6f-9e53-2fbb1b4739e6 kind: Event lastTimestamp: "2026-07-06T20:56:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:57Z" name: raw-sklearn-ef99f-predictor-77b85567df-trpsw.18bfcedb1cbd2041 namespace: kserve-ci-e2e-test resourceVersion: "20782" uid: c6996679-64a6-4731-be21-bb1228fe296c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-ef99f-predictor-77b85567df-trpsw namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: 84831045-9e8c-4f6f-9e53-2fbb1b4739e6 kind: Event lastTimestamp: "2026-07-06T20:56:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:56:57Z" name: raw-sklearn-ef99f-predictor-77b85567df-trpsw.18bfcedb1d34f1a1 namespace: kserve-ci-e2e-test resourceVersion: "20784" uid: d2d41ebc-f94f-4854-af29-23f811277da3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-06T20:56:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-ef99f-predictor-77b85567df-trpsw namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: 84831045-9e8c-4f6f-9e53-2fbb1b4739e6 kind: Event lastTimestamp: "2026-07-06T20:58:25Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:56:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:25Z" name: raw-sklearn-ef99f-predictor-77b85567df-trpsw.18bfcedb8ce0d5bc namespace: kserve-ci-e2e-test resourceVersion: "21427" uid: 017b02a5-8fc7-45d6-adf6-ac9369f3c547 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-ef99f-predictor-77b85567df-trpsw namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: 84831045-9e8c-4f6f-9e53-2fbb1b4739e6 kind: Event lastTimestamp: "2026-07-06T20:58:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T20:58:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:23Z" name: raw-sklearn-ef99f-predictor-77b85567df-trpsw.18bfceeef89b4485 namespace: kserve-ci-e2e-test resourceVersion: "21406" uid: f41ffbf1-545f-40a1-a0de-b8f98e4e810e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-ef99f-predictor-77b85567df-trpsw namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: 84831045-9e8c-4f6f-9e53-2fbb1b4739e6 kind: Event lastTimestamp: "2026-07-06T20:58:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:58:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:23Z" name: raw-sklearn-ef99f-predictor-77b85567df-trpsw.18bfceeef89b9b30 namespace: kserve-ci-e2e-test resourceVersion: "21407" uid: 7b01eae3-7b41-4e82-8852-21d94f132bda reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-ef99f-predictor-77b85567df-trpsw namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: 84831045-9e8c-4f6f-9e53-2fbb1b4739e6 kind: Event lastTimestamp: "2026-07-06T20:58:25Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:25Z" name: raw-sklearn-ef99f-predictor-77b85567df-trpsw.18bfceef931d033a namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 3f32e10b-296b-49ef-b185-813a49747f73 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-ef99f-predictor-77b85567df namespace: kserve-ci-e2e-test resourceVersion: "20701" uid: 7f56c374-518e-4ed9-ae6f-cd24b215279e kind: Event lastTimestamp: "2026-07-06T20:56:53Z" message: 'Created pod: raw-sklearn-ef99f-predictor-77b85567df-trpsw' metadata: creationTimestamp: "2026-07-06T20:56:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:56:53Z" name: raw-sklearn-ef99f-predictor-77b85567df.18bfced9f8999cd9 namespace: kserve-ci-e2e-test resourceVersion: "20711" uid: efbda7dc-2837-4257-b446-8358b13db6c1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-ef99f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20699" uid: 6815eb2f-8bb3-4ef5-a466-58e5ebc776fd kind: Event lastTimestamp: "2026-07-06T20:56:52Z" message: Scaled up replica set raw-sklearn-ef99f-predictor-77b85567df from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:56:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:56:52Z" name: raw-sklearn-ef99f-predictor.18bfced9f531a641 namespace: kserve-ci-e2e-test resourceVersion: "20702" uid: 05f26bae-b438-461a-8d9e-9bca042bd499 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:57:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-ef99f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20700" uid: d03effdd-4cf6-4f01-b7be-f087300616ce kind: Event lastTimestamp: "2026-07-06T20:57:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:57:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:57:23Z" name: raw-sklearn-ef99f-predictor.18bfcedd7480690f namespace: kserve-ci-e2e-test resourceVersion: "20964" uid: 42e20b51-9932-4cbe-8b1d-7ae5f3e7d0b5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:57:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-ef99f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20700" uid: d03effdd-4cf6-4f01-b7be-f087300616ce kind: Event lastTimestamp: "2026-07-06T20:57:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:57:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:57:23Z" name: raw-sklearn-ef99f-predictor.18bfcedd7534fba8 namespace: kserve-ci-e2e-test resourceVersion: "20965" uid: fa1b7322-d3dd-4550-9c1a-20255c2b8ed9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T20:57:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-ef99f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20856" uid: d03effdd-4cf6-4f01-b7be-f087300616ce kind: Event lastTimestamp: "2026-07-06T20:58:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:23Z" name: raw-sklearn-ef99f-predictor.18bfcee4736751ec namespace: kserve-ci-e2e-test resourceVersion: "21390" uid: 60a40769-5051-4a55-9962-0c6a7d0a2a09 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T20:57:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-ef99f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20856" uid: d03effdd-4cf6-4f01-b7be-f087300616ce kind: Event lastTimestamp: "2026-07-06T20:58:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:23Z" name: raw-sklearn-ef99f-predictor.18bfcee473dd6406 namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: ce71bf1c-38de-472a-a100-abd0c8c7c63c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-06T20:56:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-ef99f namespace: kserve-ci-e2e-test resourceVersion: "20692" uid: 39503e50-3424-4d29-9db7-0e7ed8c5ca12 kind: Event lastTimestamp: "2026-07-06T20:58:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:56:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:58:15Z" name: raw-sklearn-ef99f.18bfced9f547e658 namespace: kserve-ci-e2e-test resourceVersion: "21342" uid: b9c12121-44dd-40db-9e88-076e6ca7a2e4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:56:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-ef99f namespace: kserve-ci-e2e-test resourceVersion: "20692" uid: 39503e50-3424-4d29-9db7-0e7ed8c5ca12 kind: Event lastTimestamp: "2026-07-06T20:56:52Z" message: 'Failed to update status for InferenceService "raw-sklearn-ef99f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-ef99f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:56:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:56:52Z" name: raw-sklearn-ef99f.18bfced9f5bf114e namespace: kserve-ci-e2e-test resourceVersion: "20706" uid: 5acf06ce-c6cf-46c7-b6d5-c6b23bf96be3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-ef99f namespace: kserve-ci-e2e-test resourceVersion: "21340" uid: 39503e50-3424-4d29-9db7-0e7ed8c5ca12 kind: Event lastTimestamp: "2026-07-06T20:58:15Z" message: InferenceService [raw-sklearn-ef99f] is Ready metadata: creationTimestamp: "2026-07-06T20:58:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:58:15Z" name: raw-sklearn-ef99f.18bfceed46379de8 namespace: kserve-ci-e2e-test resourceVersion: "21341" uid: 8041390d-2149-47b1-b93d-81ad4475c91c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-06T20:58:28.364004Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m namespace: kserve-ci-e2e-test resourceVersion: "21472" uid: 891b75ec-62b9-4505-8121-e7855b6b4ab0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m to ip-10-0-139-250.ec2.internal metadata: creationTimestamp: "2026-07-06T20:58:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-06T20:58:28Z" name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m.18bfcef02a96ec08 namespace: kserve-ci-e2e-test resourceVersion: "21477" uid: f90e2903-a122-46c1-ba6e-c3e5f389904d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-95cfcc986-db5pj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:28Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m namespace: kserve-ci-e2e-test resourceVersion: "21480" uid: 891b75ec-62b9-4505-8121-e7855b6b4ab0 kind: Event lastTimestamp: "2026-07-06T20:58:28Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-06T20:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-06T20:58:28Z" name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m.18bfcef044180db5 namespace: kserve-ci-e2e-test resourceVersion: "21488" uid: 3ff8ad8f-c220-4cf3-8f95-cb06b3e87b80 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m namespace: kserve-ci-e2e-test resourceVersion: "21474" uid: 891b75ec-62b9-4505-8121-e7855b6b4ab0 kind: Event lastTimestamp: "2026-07-06T20:58:28Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:9b7f9014cc3727e4fd95a945572d3f70017252d596bd8941a1887e4a1514875c" already present on machine metadata: creationTimestamp: "2026-07-06T20:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:28Z" name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m.18bfcef04594eb0a namespace: kserve-ci-e2e-test resourceVersion: "21490" uid: 3ced5ae0-62d2-475d-a241-85bf1e0546b1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m namespace: kserve-ci-e2e-test resourceVersion: "21474" uid: 891b75ec-62b9-4505-8121-e7855b6b4ab0 kind: Event lastTimestamp: "2026-07-06T20:58:28Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-06T20:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:28Z" name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m.18bfcef0484dbb08 namespace: kserve-ci-e2e-test resourceVersion: "21491" uid: 42c1517d-c47f-45d7-8503-0a91dfc21392 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m namespace: kserve-ci-e2e-test resourceVersion: "21474" uid: 891b75ec-62b9-4505-8121-e7855b6b4ab0 kind: Event lastTimestamp: "2026-07-06T20:58:28Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-06T20:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:28Z" name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m.18bfcef048c1f813 namespace: kserve-ci-e2e-test resourceVersion: "21492" uid: 327f09b2-a881-4f9a-bdf3-fd4142ce4281 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m namespace: kserve-ci-e2e-test resourceVersion: "21474" uid: 891b75ec-62b9-4505-8121-e7855b6b4ab0 kind: Event lastTimestamp: "2026-07-06T20:58:33Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1703" already present on machine metadata: creationTimestamp: "2026-07-06T20:58:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:33Z" name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m.18bfcef1495d775f namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: 09b71618-de70-4a10-ba52-be3d23284a87 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m namespace: kserve-ci-e2e-test resourceVersion: "21474" uid: 891b75ec-62b9-4505-8121-e7855b6b4ab0 kind: Event lastTimestamp: "2026-07-06T20:58:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-06T20:58:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:33Z" name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m.18bfcef14c83e237 namespace: kserve-ci-e2e-test resourceVersion: "21529" uid: 47710fac-8d6d-4d43-ad52-b5f992eea684 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m namespace: kserve-ci-e2e-test resourceVersion: "21474" uid: 891b75ec-62b9-4505-8121-e7855b6b4ab0 kind: Event lastTimestamp: "2026-07-06T20:58:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-06T20:58:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:33Z" name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m.18bfcef14cfc1ffb namespace: kserve-ci-e2e-test resourceVersion: "21530" uid: 3db0de49-47d7-4442-b976-f504d0a6ed53 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m namespace: kserve-ci-e2e-test resourceVersion: "21474" uid: 891b75ec-62b9-4505-8121-e7855b6b4ab0 kind: Event lastTimestamp: "2026-07-06T20:58:33Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-06T20:58:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:33Z" name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m.18bfcef14d06d1e1 namespace: kserve-ci-e2e-test resourceVersion: "21531" uid: 4355d19d-4f3e-42c0-8c89-25004f15d05b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m namespace: kserve-ci-e2e-test resourceVersion: "21474" uid: 891b75ec-62b9-4505-8121-e7855b6b4ab0 kind: Event lastTimestamp: "2026-07-06T20:58:33Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-06T20:58:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:33Z" name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m.18bfcef1502b956b namespace: kserve-ci-e2e-test resourceVersion: "21534" uid: 26aed0dc-b944-47e4-be78-c08d9f2ce12f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m namespace: kserve-ci-e2e-test resourceVersion: "21474" uid: 891b75ec-62b9-4505-8121-e7855b6b4ab0 kind: Event lastTimestamp: "2026-07-06T20:58:33Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:58:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:58:33Z" name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m.18bfcef150aac2a0 namespace: kserve-ci-e2e-test resourceVersion: "21535" uid: 1555fb57-e64d-4893-9034-ebf09a0a6070 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-06T20:58:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m namespace: kserve-ci-e2e-test resourceVersion: "21474" uid: 891b75ec-62b9-4505-8121-e7855b6b4ab0 kind: Event lastTimestamp: "2026-07-06T20:59:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-06T20:58:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:59:58Z" name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m.18bfcef1c1260ab0 namespace: kserve-ci-e2e-test resourceVersion: "22176" uid: c724678a-21c9-462b-9c46-901618e2a927 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m namespace: kserve-ci-e2e-test resourceVersion: "21474" uid: 891b75ec-62b9-4505-8121-e7855b6b4ab0 kind: Event lastTimestamp: "2026-07-06T20:59:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-06T20:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:59:58Z" name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m.18bfcf0527e1a38d namespace: kserve-ci-e2e-test resourceVersion: "22171" uid: 86494975-a975-445b-998d-5eb97e8ab4f6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m namespace: kserve-ci-e2e-test resourceVersion: "21474" uid: 891b75ec-62b9-4505-8121-e7855b6b4ab0 kind: Event lastTimestamp: "2026-07-06T20:59:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-06T20:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T20:59:58Z" name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m.18bfcf0527e25533 namespace: kserve-ci-e2e-test resourceVersion: "22172" uid: bbba6a15-bc17-4a21-b10c-b49e41d68571 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T21:00:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m namespace: kserve-ci-e2e-test resourceVersion: "21474" uid: 891b75ec-62b9-4505-8121-e7855b6b4ab0 kind: Event lastTimestamp: "2026-07-06T21:00:01Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp 10.134.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-06T21:00:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-06T21:00:01Z" name: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m.18bfcf05c7545cb4 namespace: kserve-ci-e2e-test resourceVersion: "22191" uid: 52ecba4d-f539-460c-940c-d85b93c18b8f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-250.ec2.internal source: component: kubelet host: ip-10-0-139-250.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-321a7-predictor-c6d947c4f namespace: kserve-ci-e2e-test resourceVersion: "21459" uid: 9ce3f56d-cbc7-4d17-a1c9-6a6818d24f02 kind: Event lastTimestamp: "2026-07-06T20:58:28Z" message: 'Created pod: raw-sklearn-runtime-321a7-predictor-c6d947c4f-2tl5m' metadata: creationTimestamp: "2026-07-06T20:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:28Z" name: raw-sklearn-runtime-321a7-predictor-c6d947c4f.18bfcef02a19701e namespace: kserve-ci-e2e-test resourceVersion: "21473" uid: 597ce0f2-63a9-454c-a78a-5893983b57f2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-321a7-predictor namespace: kserve-ci-e2e-test resourceVersion: "21457" uid: 287ffc16-2ead-4b6d-9ef3-9a5ba3694744 kind: Event lastTimestamp: "2026-07-06T20:58:28Z" message: Scaled up replica set raw-sklearn-runtime-321a7-predictor-c6d947c4f from 0 to 1 metadata: creationTimestamp: "2026-07-06T20:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:28Z" name: raw-sklearn-runtime-321a7-predictor.18bfcef024eb6eee namespace: kserve-ci-e2e-test resourceVersion: "21462" uid: 4ac5e25e-9e83-485d-9432-2f1168785f87 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:58:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-321a7-predictor namespace: kserve-ci-e2e-test resourceVersion: "21458" uid: c8878c66-9bb9-4791-b35f-f5f765fbaf6d kind: Event lastTimestamp: "2026-07-06T20:58:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:58:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:58Z" name: raw-sklearn-runtime-321a7-predictor.18bfcef3a3fc12cf namespace: kserve-ci-e2e-test resourceVersion: "21712" uid: 5a88acca-638e-4f4b-bb59-1f30dcbdc2e4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-06T20:58:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-321a7-predictor namespace: kserve-ci-e2e-test resourceVersion: "21458" uid: c8878c66-9bb9-4791-b35f-f5f765fbaf6d kind: Event lastTimestamp: "2026-07-06T20:58:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-06T20:58:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:58:58Z" name: raw-sklearn-runtime-321a7-predictor.18bfcef3a4a5f187 namespace: kserve-ci-e2e-test resourceVersion: "21713" uid: 07e922c9-f425-419e-b19e-db5b43b67fec reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T20:59:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-321a7-predictor namespace: kserve-ci-e2e-test resourceVersion: "21614" uid: c8878c66-9bb9-4791-b35f-f5f765fbaf6d kind: Event lastTimestamp: "2026-07-06T20:59:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:59:58Z" name: raw-sklearn-runtime-321a7-predictor.18bfcefaa2e6f5c1 namespace: kserve-ci-e2e-test resourceVersion: "22155" uid: 6aff9865-d04a-4827-97a3-556f45fc82b3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-06T20:59:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-321a7-predictor namespace: kserve-ci-e2e-test resourceVersion: "21614" uid: c8878c66-9bb9-4791-b35f-f5f765fbaf6d kind: Event lastTimestamp: "2026-07-06T20:59:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-06T20:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-06T20:59:58Z" name: raw-sklearn-runtime-321a7-predictor.18bfcefaa3592849 namespace: kserve-ci-e2e-test resourceVersion: "22156" uid: e295e536-dd6e-4809-8443-dd3d98812dcf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-06T20:58:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-321a7 namespace: kserve-ci-e2e-test resourceVersion: "21449" uid: b3b0d81e-8d8d-4a05-b6de-2a4cb7e7e0a7 kind: Event lastTimestamp: "2026-07-06T20:59:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-06T20:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:59:49Z" name: raw-sklearn-runtime-321a7.18bfcef024e3716e namespace: kserve-ci-e2e-test resourceVersion: "22081" uid: 30e98dbf-b072-472c-a6d6-7ec7c34c4fcc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:58:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-321a7 namespace: kserve-ci-e2e-test resourceVersion: "21449" uid: b3b0d81e-8d8d-4a05-b6de-2a4cb7e7e0a7 kind: Event lastTimestamp: "2026-07-06T20:58:28Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-321a7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-321a7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-06T20:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:58:28Z" name: raw-sklearn-runtime-321a7.18bfcef02589e67e namespace: kserve-ci-e2e-test resourceVersion: "21465" uid: c0a6b9ed-3dab-4da7-988f-ce7a9b2f8c81 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-06T20:59:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-321a7 namespace: kserve-ci-e2e-test resourceVersion: "22078" uid: b3b0d81e-8d8d-4a05-b6de-2a4cb7e7e0a7 kind: Event lastTimestamp: "2026-07-06T20:59:48Z" message: InferenceService [raw-sklearn-runtime-321a7] is Ready metadata: creationTimestamp: "2026-07-06T20:59:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-06T20:59:48Z" name: raw-sklearn-runtime-321a7.18bfcf02ee205de0 namespace: kserve-ci-e2e-test resourceVersion: "22079" uid: f0bd5b32-9ac8-4b57-a5af-cb007954a845 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "22425"