--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:50:14.432408Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-eec619-predictor-6bd958d4dc-zbwhq namespace: kserve-ci-e2e-test resourceVersion: "18843" uid: 00027eda-57a3-456a-832a-433dd30691fc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-eec619-predictor-6bd958d4dc-zbwhq to ip-10-0-140-154.ec2.internal metadata: creationTimestamp: "2026-05-20T15:50:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:50:14Z" name: isvc-init-fail-eec619-predictor-6bd958d4dc-zbwhq.18b150d788dccf07 namespace: kserve-ci-e2e-test resourceVersion: "18847" uid: b5964025-ef46-4ad2-8f1f-32f55d2ef3ce reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df74f7774-d6kd2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:50:14Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-eec619-predictor-6bd958d4dc-zbwhq namespace: kserve-ci-e2e-test resourceVersion: "18853" uid: 00027eda-57a3-456a-832a-433dd30691fc kind: Event lastTimestamp: "2026-05-20T15:50:14Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:50:14Z" name: isvc-init-fail-eec619-predictor-6bd958d4dc-zbwhq.18b150d7a460a6cf namespace: kserve-ci-e2e-test resourceVersion: "18865" uid: ed5333c6-80eb-4b57-a384-70aabbb6aff2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-eec619-predictor-6bd958d4dc-zbwhq namespace: kserve-ci-e2e-test resourceVersion: "18845" uid: 00027eda-57a3-456a-832a-433dd30691fc kind: Event lastTimestamp: "2026-05-20T15:50:22Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:050b18baea3a969ddc2a15671f898a5f40ff53eb89f54c18856b130dc10bd894" already present on machine metadata: creationTimestamp: "2026-05-20T15:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:50:22Z" name: isvc-init-fail-eec619-predictor-6bd958d4dc-zbwhq.18b150d7a5d2355a namespace: kserve-ci-e2e-test resourceVersion: "18935" uid: dfca2763-5d98-45f6-a2f3-023b6647fd2e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-eec619-predictor-6bd958d4dc-zbwhq namespace: kserve-ci-e2e-test resourceVersion: "18845" uid: 00027eda-57a3-456a-832a-433dd30691fc kind: Event lastTimestamp: "2026-05-20T15:50:22Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T15:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:50:22Z" name: isvc-init-fail-eec619-predictor-6bd958d4dc-zbwhq.18b150d7a9168cbf namespace: kserve-ci-e2e-test resourceVersion: "18938" uid: 6861d61c-b462-4ae8-a49f-8d30968ad12f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-eec619-predictor-6bd958d4dc-zbwhq namespace: kserve-ci-e2e-test resourceVersion: "18845" uid: 00027eda-57a3-456a-832a-433dd30691fc kind: Event lastTimestamp: "2026-05-20T15:50:22Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T15:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:50:22Z" name: isvc-init-fail-eec619-predictor-6bd958d4dc-zbwhq.18b150d7a994e0a4 namespace: kserve-ci-e2e-test resourceVersion: "18941" uid: b3d4ae12-436d-4a77-9e89-39ae27953159 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:50:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-eec619-predictor-6bd958d4dc-zbwhq namespace: kserve-ci-e2e-test resourceVersion: "18845" uid: 00027eda-57a3-456a-832a-433dd30691fc kind: Event lastTimestamp: "2026-05-20T15:50:24Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-05-20T15:50:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:50:24Z" name: isvc-init-fail-eec619-predictor-6bd958d4dc-zbwhq.18b150d9f4c2152a namespace: kserve-ci-e2e-test resourceVersion: "18996" uid: 366bf3ea-8160-43cf-9f46-0abcce08ecab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:50:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-eec619-predictor-6bd958d4dc namespace: kserve-ci-e2e-test resourceVersion: "18831" uid: e10c0954-3426-43da-a6a0-53b6ef490af8 kind: Event lastTimestamp: "2026-05-20T15:50:14Z" message: 'Created pod: isvc-init-fail-eec619-predictor-6bd958d4dc-zbwhq' metadata: creationTimestamp: "2026-05-20T15:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:50:14Z" name: isvc-init-fail-eec619-predictor-6bd958d4dc.18b150d7886a5e55 namespace: kserve-ci-e2e-test resourceVersion: "18844" uid: e4ece43b-7432-4a35-8417-4dfc063378cf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:50:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-eec619-predictor namespace: kserve-ci-e2e-test resourceVersion: "18825" uid: 0ecd1080-956c-4b80-9ec8-df043ac83e15 kind: Event lastTimestamp: "2026-05-20T15:50:14Z" message: Scaled up replica set isvc-init-fail-eec619-predictor-6bd958d4dc from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:50:14Z" name: isvc-init-fail-eec619-predictor.18b150d784fea0c8 namespace: kserve-ci-e2e-test resourceVersion: "18833" uid: 0bc5174a-8708-49cf-b090-08acf822c5b4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-20T15:50:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-eec619 namespace: kserve-ci-e2e-test resourceVersion: "18796" uid: a8faafa8-69a3-4c3c-bb93-6ff3a022e71f kind: Event lastTimestamp: "2026-05-20T15:50:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:50:22Z" name: isvc-init-fail-eec619.18b150d784d60bf7 namespace: kserve-ci-e2e-test resourceVersion: "18942" uid: a3617140-fd2a-4271-b6a3-2c9d837fff96 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:50:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-eec619 namespace: kserve-ci-e2e-test resourceVersion: "18796" uid: a8faafa8-69a3-4c3c-bb93-6ff3a022e71f kind: Event lastTimestamp: "2026-05-20T15:50:14Z" message: 'Failed to update status for InferenceService "isvc-init-fail-eec619": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-eec619": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:50:14Z" name: isvc-init-fail-eec619.18b150d7857316ee namespace: kserve-ci-e2e-test resourceVersion: "18836" uid: b262ce20-f5ba-41e4-9393-4b65f82d3f79 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:50:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-eec619 namespace: kserve-ci-e2e-test resourceVersion: "18796" uid: a8faafa8-69a3-4c3c-bb93-6ff3a022e71f kind: Event lastTimestamp: "2026-05-20T15:50:14Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-eec619": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:50:14Z" name: isvc-init-fail-eec619.18b150d785734d62 namespace: kserve-ci-e2e-test resourceVersion: "18837" uid: 461000e5-2930-4f15-b8c5-b21971363fef reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:44:40.602915Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: 1c06339d-3a70-49f8-8ecf-70aa9c8c30d6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-47eff-predictor-5478d4946-jtzdg to ip-10-0-140-154.ec2.internal metadata: creationTimestamp: "2026-05-20T15:44:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:44:40Z" name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg.18b15089cf130a60 namespace: kserve-ci-e2e-test resourceVersion: "16070" uid: 2771aa0a-6902-4c4e-833c-844ca53e8718 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df74f7774-d6kd2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:41Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: 1c06339d-3a70-49f8-8ecf-70aa9c8c30d6 kind: Event lastTimestamp: "2026-05-20T15:44:41Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:44:41Z" name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg.18b15089e9038b11 namespace: kserve-ci-e2e-test resourceVersion: "16082" uid: 134aa531-34ce-482c-a6f1-1438751a6344 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 1c06339d-3a70-49f8-8ecf-70aa9c8c30d6 kind: Event lastTimestamp: "2026-05-20T15:44:41Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:050b18baea3a969ddc2a15671f898a5f40ff53eb89f54c18856b130dc10bd894" already present on machine metadata: creationTimestamp: "2026-05-20T15:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:41Z" name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg.18b15089eab89d78 namespace: kserve-ci-e2e-test resourceVersion: "16085" uid: c653943b-e391-46f1-ab68-65555136eca9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 1c06339d-3a70-49f8-8ecf-70aa9c8c30d6 kind: Event lastTimestamp: "2026-05-20T15:44:41Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T15:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:41Z" name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg.18b15089ee20b295 namespace: kserve-ci-e2e-test resourceVersion: "16086" uid: 64c7e3ed-e1d6-47c9-9ef1-812629260ddc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 1c06339d-3a70-49f8-8ecf-70aa9c8c30d6 kind: Event lastTimestamp: "2026-05-20T15:44:41Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T15:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:41Z" name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg.18b15089eea04024 namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: 1e1c17f3-7a6e-44a9-977b-9b0076f5b051 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 1c06339d-3a70-49f8-8ecf-70aa9c8c30d6 kind: Event lastTimestamp: "2026-05-20T15:44:45Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-05-20T15:44:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:45Z" name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg.18b1508afab3003a namespace: kserve-ci-e2e-test resourceVersion: "16128" uid: 57452813-0b5b-4e67-b46f-503a14838726 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 1c06339d-3a70-49f8-8ecf-70aa9c8c30d6 kind: Event lastTimestamp: "2026-05-20T15:44:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:44:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:45Z" name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg.18b1508afed718d1 namespace: kserve-ci-e2e-test resourceVersion: "16133" uid: 4e77d95a-bcc7-4594-9ba0-48ffca8d6851 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 1c06339d-3a70-49f8-8ecf-70aa9c8c30d6 kind: Event lastTimestamp: "2026-05-20T15:44:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:44:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:45Z" name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg.18b1508aff4b9483 namespace: kserve-ci-e2e-test resourceVersion: "16134" uid: 732793eb-1ce1-4b08-8d6d-0838de2cf0e2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 1c06339d-3a70-49f8-8ecf-70aa9c8c30d6 kind: Event lastTimestamp: "2026-05-20T15:44:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T15:44:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:45Z" name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg.18b1508aff5932cc namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: ff98e2c7-62f8-4077-9f7c-4b68bd232b95 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 1c06339d-3a70-49f8-8ecf-70aa9c8c30d6 kind: Event lastTimestamp: "2026-05-20T15:44:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:44:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:45Z" name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg.18b1508b027fa7c9 namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 9535e0ba-2944-4590-bd62-27aebbaef059 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 1c06339d-3a70-49f8-8ecf-70aa9c8c30d6 kind: Event lastTimestamp: "2026-05-20T15:44:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:44:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:45Z" name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg.18b1508b02ec60f7 namespace: kserve-ci-e2e-test resourceVersion: "16138" uid: b2567e09-a249-4709-80ed-9b022ad5a038 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 1c06339d-3a70-49f8-8ecf-70aa9c8c30d6 kind: Event lastTimestamp: "2026-05-20T15:44:45Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:e9391ee1281af7f81a916bc80b584691bef158df8f07d34a6fb179b3bd4767e7" already present on machine metadata: creationTimestamp: "2026-05-20T15:44:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:45Z" name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg.18b1508b02f739e6 namespace: kserve-ci-e2e-test resourceVersion: "16140" uid: 0c2e7dae-6a50-4f95-8983-b41c3632f25c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 1c06339d-3a70-49f8-8ecf-70aa9c8c30d6 kind: Event lastTimestamp: "2026-05-20T15:44:45Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-05-20T15:44:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:45Z" name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg.18b1508b05ed95da namespace: kserve-ci-e2e-test resourceVersion: "16141" uid: 7a581bf5-9cb5-4c9f-9f48-93927ad9d4b1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 1c06339d-3a70-49f8-8ecf-70aa9c8c30d6 kind: Event lastTimestamp: "2026-05-20T15:44:45Z" message: Started container agent metadata: creationTimestamp: "2026-05-20T15:44:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:45Z" name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg.18b1508b065f9e3a namespace: kserve-ci-e2e-test resourceVersion: "16142" uid: c480c5b5-0ada-4f2b-bd96-968c451b83ac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-05-20T15:44:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 1c06339d-3a70-49f8-8ecf-70aa9c8c30d6 kind: Event lastTimestamp: "2026-05-20T15:46:23Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:46:23Z" name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg.18b1508b36ad4c73 namespace: kserve-ci-e2e-test resourceVersion: "16972" uid: 78f37e2d-dd15-425b-b9f7-17a2f5b9478f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-20T15:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 1c06339d-3a70-49f8-8ecf-70aa9c8c30d6 kind: Event lastTimestamp: "2026-05-20T15:46:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T15:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:46:23Z" name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg.18b1508b727a02dc namespace: kserve-ci-e2e-test resourceVersion: "16973" uid: 5ee2224e-0f12-4154-b238-e411ffd90dc7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 1c06339d-3a70-49f8-8ecf-70aa9c8c30d6 kind: Event lastTimestamp: "2026-05-20T15:46:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T15:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:46:05Z" name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg.18b1509da2548fb1 namespace: kserve-ci-e2e-test resourceVersion: "16769" uid: 4a84578b-7e72-445b-a490-4494ebe0ab37 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 1c06339d-3a70-49f8-8ecf-70aa9c8c30d6 kind: Event lastTimestamp: "2026-05-20T15:46:05Z" message: Stopping container agent metadata: creationTimestamp: "2026-05-20T15:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:46:05Z" name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg.18b1509da254cd85 namespace: kserve-ci-e2e-test resourceVersion: "16774" uid: 80ec9b27-cd20-4b51-951c-568ac36624f5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 1c06339d-3a70-49f8-8ecf-70aa9c8c30d6 kind: Event lastTimestamp: "2026-05-20T15:46:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:46:05Z" name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg.18b1509da254feab namespace: kserve-ci-e2e-test resourceVersion: "16773" uid: 63057412-dc16-4dc0-ac5f-ef08f581f242 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:46:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 1c06339d-3a70-49f8-8ecf-70aa9c8c30d6 kind: Event lastTimestamp: "2026-05-20T15:46:23Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:46:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:46:23Z" name: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg.18b1509e4e9e0dbe namespace: kserve-ci-e2e-test resourceVersion: "16971" uid: 87404a53-038d-4b2c-955b-c34aefc989ca reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-47eff-predictor-5478d4946 namespace: kserve-ci-e2e-test resourceVersion: "16053" uid: 400965fb-58eb-4f9a-8517-72a0b1ee7328 kind: Event lastTimestamp: "2026-05-20T15:44:40Z" message: 'Created pod: isvc-logger-raw-47eff-predictor-5478d4946-jtzdg' metadata: creationTimestamp: "2026-05-20T15:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:40Z" name: isvc-logger-raw-47eff-predictor-5478d4946.18b15089ce43957c namespace: kserve-ci-e2e-test resourceVersion: "16067" uid: 61e9e471-f75f-47db-a151-ae98469bc560 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-47eff-predictor namespace: kserve-ci-e2e-test resourceVersion: "16052" uid: 3f43945a-b40a-4e47-af50-800e5404b70a kind: Event lastTimestamp: "2026-05-20T15:44:40Z" message: Scaled up replica set isvc-logger-raw-47eff-predictor-5478d4946 from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:40Z" name: isvc-logger-raw-47eff-predictor.18b15089c938177a namespace: kserve-ci-e2e-test resourceVersion: "16055" uid: b2a16110-6607-493a-83fe-3b5cb1053bb0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:44:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-47eff-predictor namespace: kserve-ci-e2e-test resourceVersion: "16054" uid: 6b4e3b3e-96bb-41dc-bd4f-9ae93bddf0c3 kind: Event lastTimestamp: "2026-05-20T15:45:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:44:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:25Z" name: isvc-logger-raw-47eff-predictor.18b1508d480a9d0a namespace: kserve-ci-e2e-test resourceVersion: "16440" uid: d1102bae-e737-4b02-beea-b8baedf82d2d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:44:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-47eff-predictor namespace: kserve-ci-e2e-test resourceVersion: "16054" uid: 6b4e3b3e-96bb-41dc-bd4f-9ae93bddf0c3 kind: Event lastTimestamp: "2026-05-20T15:45:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:44:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:25Z" name: isvc-logger-raw-47eff-predictor.18b1508d487916b5 namespace: kserve-ci-e2e-test resourceVersion: "16441" uid: c08a6947-b0cc-4a62-a223-f26ef6b742a9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:45:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-47eff-predictor namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: 6b4e3b3e-96bb-41dc-bd4f-9ae93bddf0c3 kind: Event lastTimestamp: "2026-05-20T15:45:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:55Z" name: isvc-logger-raw-47eff-predictor.18b15090c7dcb638 namespace: kserve-ci-e2e-test resourceVersion: "16665" uid: 37827237-21da-4684-8f07-f228452a598a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:45:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-47eff-predictor namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: 6b4e3b3e-96bb-41dc-bd4f-9ae93bddf0c3 kind: Event lastTimestamp: "2026-05-20T15:45:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:55Z" name: isvc-logger-raw-47eff-predictor.18b15090c86ec00f namespace: kserve-ci-e2e-test resourceVersion: "16666" uid: 8e0ecb5c-c54a-471f-8b00-6712943400ee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-05-20T15:44:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-47eff namespace: kserve-ci-e2e-test resourceVersion: "16042" uid: 160669f4-1062-4c65-b556-653fb90926c7 kind: Event lastTimestamp: "2026-05-20T15:45:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:45:53Z" name: isvc-logger-raw-47eff.18b15089c9445987 namespace: kserve-ci-e2e-test resourceVersion: "16651" uid: 99754c49-66a2-4d24-bfe5-6cd9adbe4c6c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-47eff namespace: kserve-ci-e2e-test resourceVersion: "16042" uid: 160669f4-1062-4c65-b556-653fb90926c7 kind: Event lastTimestamp: "2026-05-20T15:44:40Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-47eff": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-47eff": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:44:40Z" name: isvc-logger-raw-47eff.18b15089c9dc086a namespace: kserve-ci-e2e-test resourceVersion: "16058" uid: e9976310-25c6-48db-b7ef-b3f1700212af reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-47eff namespace: kserve-ci-e2e-test resourceVersion: "16647" uid: 160669f4-1062-4c65-b556-653fb90926c7 kind: Event lastTimestamp: "2026-05-20T15:45:53Z" message: InferenceService [isvc-logger-raw-47eff] is Ready metadata: creationTimestamp: "2026-05-20T15:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:45:53Z" name: isvc-logger-raw-47eff.18b1509ad7fe203d namespace: kserve-ci-e2e-test resourceVersion: "16649" uid: 601c2050-edf9-47ba-b139-aa952303944f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:48:36.141840Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4 namespace: kserve-ci-e2e-test resourceVersion: "17965" uid: 3425f3d9-1734-4ad9-bec4-d94e85c206ec kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-c3463e-predictor-86cdb4986d-qqmd4 to ip-10-0-140-154.ec2.internal metadata: creationTimestamp: "2026-05-20T15:48:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:48:36Z" name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4.18b150c0a649bc2a namespace: kserve-ci-e2e-test resourceVersion: "17970" uid: 29f9d6d7-95cb-469b-ba88-aa005d1b7745 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df74f7774-d6kd2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:48:36Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4 namespace: kserve-ci-e2e-test resourceVersion: "17976" uid: 3425f3d9-1734-4ad9-bec4-d94e85c206ec kind: Event lastTimestamp: "2026-05-20T15:48:36Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:48:36Z" name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4.18b150c0c0944793 namespace: kserve-ci-e2e-test resourceVersion: "17985" uid: 2a4c5eb8-f307-49a3-b28c-e14c61bc2828 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:48:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4 namespace: kserve-ci-e2e-test resourceVersion: "17967" uid: 3425f3d9-1734-4ad9-bec4-d94e85c206ec kind: Event lastTimestamp: "2026-05-20T15:48:36Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:050b18baea3a969ddc2a15671f898a5f40ff53eb89f54c18856b130dc10bd894" already present on machine metadata: creationTimestamp: "2026-05-20T15:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:48:36Z" name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4.18b150c0c21d82f5 namespace: kserve-ci-e2e-test resourceVersion: "17987" uid: 51758fa8-5a87-4ab5-8127-b557787f798e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:48:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4 namespace: kserve-ci-e2e-test resourceVersion: "17967" uid: 3425f3d9-1734-4ad9-bec4-d94e85c206ec kind: Event lastTimestamp: "2026-05-20T15:48:36Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T15:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:48:36Z" name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4.18b150c0c4eef5ae namespace: kserve-ci-e2e-test resourceVersion: "17988" uid: 9e310007-f5bd-46db-9790-00b7ef241054 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:48:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4 namespace: kserve-ci-e2e-test resourceVersion: "17967" uid: 3425f3d9-1734-4ad9-bec4-d94e85c206ec kind: Event lastTimestamp: "2026-05-20T15:48:36Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T15:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:48:36Z" name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4.18b150c0c56c52c9 namespace: kserve-ci-e2e-test resourceVersion: "17989" uid: 99ad275e-84cb-4337-ad1a-978dfd96d925 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:48:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4 namespace: kserve-ci-e2e-test resourceVersion: "17967" uid: 3425f3d9-1734-4ad9-bec4-d94e85c206ec kind: Event lastTimestamp: "2026-05-20T15:48:41Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-05-20T15:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:48:41Z" name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4.18b150c1e306c578 namespace: kserve-ci-e2e-test resourceVersion: "18030" uid: 3952ff0a-8dc2-4638-a749-8f472fa7b99a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:48:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4 namespace: kserve-ci-e2e-test resourceVersion: "17967" uid: 3425f3d9-1734-4ad9-bec4-d94e85c206ec kind: Event lastTimestamp: "2026-05-20T15:48:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:48:41Z" name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4.18b150c1e60ce416 namespace: kserve-ci-e2e-test resourceVersion: "18032" uid: 1c8eb6ba-c62f-440e-958e-b4a9961cb655 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:48:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4 namespace: kserve-ci-e2e-test resourceVersion: "17967" uid: 3425f3d9-1734-4ad9-bec4-d94e85c206ec kind: Event lastTimestamp: "2026-05-20T15:48:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:48:41Z" name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4.18b150c1e679724d namespace: kserve-ci-e2e-test resourceVersion: "18034" uid: bec46e63-df36-4170-9e6c-8c0b8651a7b6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:48:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4 namespace: kserve-ci-e2e-test resourceVersion: "17967" uid: 3425f3d9-1734-4ad9-bec4-d94e85c206ec kind: Event lastTimestamp: "2026-05-20T15:48:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T15:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:48:41Z" name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4.18b150c1e685d23e namespace: kserve-ci-e2e-test resourceVersion: "18036" uid: 4edcaead-72ac-40a5-a08a-04d94df17e99 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:48:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4 namespace: kserve-ci-e2e-test resourceVersion: "17967" uid: 3425f3d9-1734-4ad9-bec4-d94e85c206ec kind: Event lastTimestamp: "2026-05-20T15:48:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:48:41Z" name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4.18b150c1e910198d namespace: kserve-ci-e2e-test resourceVersion: "18038" uid: 8e1cefc1-6791-4b4e-b8cc-1b34d2862fdc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:48:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4 namespace: kserve-ci-e2e-test resourceVersion: "17967" uid: 3425f3d9-1734-4ad9-bec4-d94e85c206ec kind: Event lastTimestamp: "2026-05-20T15:48:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:48:41Z" name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4.18b150c1e97da5f3 namespace: kserve-ci-e2e-test resourceVersion: "18039" uid: b9f08390-56cf-4740-986c-5a5aeb31a680 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-20T15:48:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4 namespace: kserve-ci-e2e-test resourceVersion: "17967" uid: 3425f3d9-1734-4ad9-bec4-d94e85c206ec kind: Event lastTimestamp: "2026-05-20T15:49:39Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:48:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:49:39Z" name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4.18b150c25ac7ff5b namespace: kserve-ci-e2e-test resourceVersion: "18458" uid: 5891fe44-4941-489b-9218-5274db25dbf5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4 namespace: kserve-ci-e2e-test resourceVersion: "17967" uid: 3425f3d9-1734-4ad9-bec4-d94e85c206ec kind: Event lastTimestamp: "2026-05-20T15:50:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T15:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:50:14Z" name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4.18b150d785d927e7 namespace: kserve-ci-e2e-test resourceVersion: "18838" uid: cbb4377b-12ff-4e2c-ae92-70021e6a49cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4 namespace: kserve-ci-e2e-test resourceVersion: "17967" uid: 3425f3d9-1734-4ad9-bec4-d94e85c206ec kind: Event lastTimestamp: "2026-05-20T15:50:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:50:14Z" name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4.18b150d785d9a6b2 namespace: kserve-ci-e2e-test resourceVersion: "18840" uid: a9adeb94-dae9-4c01-abea-bfc02441bbb6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4 namespace: kserve-ci-e2e-test resourceVersion: "17967" uid: 3425f3d9-1734-4ad9-bec4-d94e85c206ec kind: Event lastTimestamp: "2026-05-20T15:50:14Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:50:14Z" name: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4.18b150d78af5ae07 namespace: kserve-ci-e2e-test resourceVersion: "18854" uid: 0e43c5ab-ecf9-4b1b-be1e-cbdd309bd5ee reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:48:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-c3463e-predictor-86cdb4986d namespace: kserve-ci-e2e-test resourceVersion: "17956" uid: 69df28e5-e63d-4330-b5d2-d3932282c9d0 kind: Event lastTimestamp: "2026-05-20T15:48:36Z" message: 'Created pod: isvc-primary-c3463e-predictor-86cdb4986d-qqmd4' metadata: creationTimestamp: "2026-05-20T15:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:48:36Z" name: isvc-primary-c3463e-predictor-86cdb4986d.18b150c0a5d454da namespace: kserve-ci-e2e-test resourceVersion: "17966" uid: 9da023dc-f188-4fd3-81ce-811feb943c09 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:48:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-c3463e-predictor namespace: kserve-ci-e2e-test resourceVersion: "17954" uid: 482be9f7-b235-4e50-9be6-246f0469cafc kind: Event lastTimestamp: "2026-05-20T15:48:36Z" message: Scaled up replica set isvc-primary-c3463e-predictor-86cdb4986d from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:48:36Z" name: isvc-primary-c3463e-predictor.18b150c09f9de47f namespace: kserve-ci-e2e-test resourceVersion: "17959" uid: 537001ee-f1eb-4cf2-8825-335c06a9001b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:48:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-c3463e-predictor namespace: kserve-ci-e2e-test resourceVersion: "17955" uid: 80580fb2-6665-4e86-bf60-87a3885ce2bf kind: Event lastTimestamp: "2026-05-20T15:48:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:48:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:48:51Z" name: isvc-primary-c3463e-predictor.18b150c41edb92d9 namespace: kserve-ci-e2e-test resourceVersion: "18114" uid: 28823a75-c290-47d7-a8a7-3483b440dc15 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:48:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-c3463e-predictor namespace: kserve-ci-e2e-test resourceVersion: "17955" uid: 80580fb2-6665-4e86-bf60-87a3885ce2bf kind: Event lastTimestamp: "2026-05-20T15:48:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:48:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:48:51Z" name: isvc-primary-c3463e-predictor.18b150c41f44b1e3 namespace: kserve-ci-e2e-test resourceVersion: "18115" uid: fbdec447-551f-4858-b7ff-3b45615e48ab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T15:49:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-c3463e-predictor namespace: kserve-ci-e2e-test resourceVersion: "18113" uid: 80580fb2-6665-4e86-bf60-87a3885ce2bf kind: Event lastTimestamp: "2026-05-20T15:50:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:49:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:50:06Z" name: isvc-primary-c3463e-predictor.18b150c79e2d1cd2 namespace: kserve-ci-e2e-test resourceVersion: "18711" uid: 7a61299b-a0dc-4627-bb6a-d90daa4c58ea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T15:49:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-c3463e-predictor namespace: kserve-ci-e2e-test resourceVersion: "18113" uid: 80580fb2-6665-4e86-bf60-87a3885ce2bf kind: Event lastTimestamp: "2026-05-20T15:50:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:49:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:50:06Z" name: isvc-primary-c3463e-predictor.18b150c79e807263 namespace: kserve-ci-e2e-test resourceVersion: "18712" uid: d4a2ab06-09d2-4a0b-a1bd-4c0ba6ad5106 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-05-20T15:48:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-c3463e namespace: kserve-ci-e2e-test resourceVersion: "17924" uid: 6e0c10b8-f217-41f6-98e0-f9d60f39c31d kind: Event lastTimestamp: "2026-05-20T15:49:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:49:49Z" name: isvc-primary-c3463e.18b150c09f2efde8 namespace: kserve-ci-e2e-test resourceVersion: "18535" uid: 26e92f7c-dac7-4e56-928c-ce5e7defea84 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:48:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-c3463e namespace: kserve-ci-e2e-test resourceVersion: "17924" uid: 6e0c10b8-f217-41f6-98e0-f9d60f39c31d kind: Event lastTimestamp: "2026-05-20T15:48:36Z" message: 'Failed to update status for InferenceService "isvc-primary-c3463e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-c3463e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:48:36Z" name: isvc-primary-c3463e.18b150c09fad3ede namespace: kserve-ci-e2e-test resourceVersion: "17973" uid: 567d294d-e6ab-4fe4-a671-baa2382a5dfd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:48:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-c3463e namespace: kserve-ci-e2e-test resourceVersion: "17924" uid: 6e0c10b8-f217-41f6-98e0-f9d60f39c31d kind: Event lastTimestamp: "2026-05-20T15:48:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-c3463e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:48:36Z" name: isvc-primary-c3463e.18b150c09fad92dd namespace: kserve-ci-e2e-test resourceVersion: "17975" uid: bf8c7334-7a17-46c7-b0f0-33ddf4db1031 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:49:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-c3463e namespace: kserve-ci-e2e-test resourceVersion: "18533" uid: 6e0c10b8-f217-41f6-98e0-f9d60f39c31d kind: Event lastTimestamp: "2026-05-20T15:49:49Z" message: InferenceService [isvc-primary-c3463e] is Ready metadata: creationTimestamp: "2026-05-20T15:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:49:49Z" name: isvc-primary-c3463e.18b150d1bec60d61 namespace: kserve-ci-e2e-test resourceVersion: "18534" uid: a19ef65a-bd06-414a-9a54-7533952c5bdd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:39:14.680367Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw namespace: kserve-ci-e2e-test resourceVersion: "12987" uid: 28616225-17ab-40a2-8c93-3f711b09517d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw to ip-10-0-140-154.ec2.internal metadata: creationTimestamp: "2026-05-20T15:39:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:39:14Z" name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw.18b1503dec93901a namespace: kserve-ci-e2e-test resourceVersion: "12993" uid: ac9c4970-13ab-4fee-956b-34204dd53ee7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df74f7774-d6kd2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:39:14Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw namespace: kserve-ci-e2e-test resourceVersion: "12990" uid: 28616225-17ab-40a2-8c93-3f711b09517d kind: Event lastTimestamp: "2026-05-20T15:39:14Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-raw-sklearn-batcher-be7d1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T15:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:39:14Z" name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw.18b1503df8a53085 namespace: kserve-ci-e2e-test resourceVersion: "13003" uid: d84f3281-855a-4aa3-85e5-2211aa93a018 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:39:15Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw namespace: kserve-ci-e2e-test resourceVersion: "12998" uid: 28616225-17ab-40a2-8c93-3f711b09517d kind: Event lastTimestamp: "2026-05-20T15:39:15Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:39:15Z" name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw.18b1503e29ae5da2 namespace: kserve-ci-e2e-test resourceVersion: "13011" uid: 923a32ad-2e4f-486a-a038-09b9c1396b75 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:39:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw namespace: kserve-ci-e2e-test resourceVersion: "12990" uid: 28616225-17ab-40a2-8c93-3f711b09517d kind: Event lastTimestamp: "2026-05-20T15:39:15Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:050b18baea3a969ddc2a15671f898a5f40ff53eb89f54c18856b130dc10bd894" metadata: creationTimestamp: "2026-05-20T15:39:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:39:15Z" name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw.18b1503e2ba70e90 namespace: kserve-ci-e2e-test resourceVersion: "13013" uid: 1fdcc4b0-f57f-48e7-a36b-1df4ddf9705d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw namespace: kserve-ci-e2e-test resourceVersion: "12990" uid: 28616225-17ab-40a2-8c93-3f711b09517d kind: Event lastTimestamp: "2026-05-20T15:39:19Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:050b18baea3a969ddc2a15671f898a5f40ff53eb89f54c18856b130dc10bd894" in 3.668s (3.668s including waiting). Image size: 301471312 bytes.' metadata: creationTimestamp: "2026-05-20T15:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:39:19Z" name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw.18b1503f06494ec6 namespace: kserve-ci-e2e-test resourceVersion: "13040" uid: 3afcef43-bfd1-4b5e-9171-3016345551b2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw namespace: kserve-ci-e2e-test resourceVersion: "12990" uid: 28616225-17ab-40a2-8c93-3f711b09517d kind: Event lastTimestamp: "2026-05-20T15:39:19Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T15:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:39:19Z" name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw.18b1503f0a8efaf5 namespace: kserve-ci-e2e-test resourceVersion: "13041" uid: f8251b37-3eb2-49f6-ae70-b98de8e7d25f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw namespace: kserve-ci-e2e-test resourceVersion: "12990" uid: 28616225-17ab-40a2-8c93-3f711b09517d kind: Event lastTimestamp: "2026-05-20T15:39:19Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T15:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:39:19Z" name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw.18b1503f0b19710c namespace: kserve-ci-e2e-test resourceVersion: "13042" uid: 98427153-7806-48ba-a049-27e14df2e9fc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:39:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw namespace: kserve-ci-e2e-test resourceVersion: "12990" uid: 28616225-17ab-40a2-8c93-3f711b09517d kind: Event lastTimestamp: "2026-05-20T15:39:24Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1433" metadata: creationTimestamp: "2026-05-20T15:39:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:39:24Z" name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw.18b150402fa3755b namespace: kserve-ci-e2e-test resourceVersion: "13087" uid: 8c42b37b-a718-491c-86bc-afeba76c6df8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:39:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw namespace: kserve-ci-e2e-test resourceVersion: "12990" uid: 28616225-17ab-40a2-8c93-3f711b09517d kind: Event lastTimestamp: "2026-05-20T15:39:37Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1433" in 12.713s (12.713s including waiting). Image size: 1560928674 bytes.' metadata: creationTimestamp: "2026-05-20T15:39:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:39:37Z" name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw.18b1504325716f4d namespace: kserve-ci-e2e-test resourceVersion: "13210" uid: dd6cbf0d-219b-4de1-a0b9-439f4dd13638 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:39:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw namespace: kserve-ci-e2e-test resourceVersion: "12990" uid: 28616225-17ab-40a2-8c93-3f711b09517d kind: Event lastTimestamp: "2026-05-20T15:39:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:39:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:39:37Z" name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw.18b150432911ac76 namespace: kserve-ci-e2e-test resourceVersion: "13211" uid: f486384c-09c9-441d-8d03-4117984eadec reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:39:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw namespace: kserve-ci-e2e-test resourceVersion: "12990" uid: 28616225-17ab-40a2-8c93-3f711b09517d kind: Event lastTimestamp: "2026-05-20T15:39:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:39:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:39:37Z" name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw.18b1504329930c43 namespace: kserve-ci-e2e-test resourceVersion: "13212" uid: 99018066-249e-47f7-980f-ad5c0776802d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:39:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw namespace: kserve-ci-e2e-test resourceVersion: "12990" uid: 28616225-17ab-40a2-8c93-3f711b09517d kind: Event lastTimestamp: "2026-05-20T15:39:37Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-20T15:39:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:39:37Z" name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw.18b1504329a0887f namespace: kserve-ci-e2e-test resourceVersion: "13214" uid: f64e23bc-20df-4bf9-8099-118c69738c0f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:39:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw namespace: kserve-ci-e2e-test resourceVersion: "12990" uid: 28616225-17ab-40a2-8c93-3f711b09517d kind: Event lastTimestamp: "2026-05-20T15:39:39Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.23s (2.23s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-20T15:39:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:39:39Z" name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw.18b15043ae9014a7 namespace: kserve-ci-e2e-test resourceVersion: "13253" uid: 008b30ea-277a-470c-8201-8b5bfc427b65 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:39:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw namespace: kserve-ci-e2e-test resourceVersion: "12990" uid: 28616225-17ab-40a2-8c93-3f711b09517d kind: Event lastTimestamp: "2026-05-20T15:39:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:39:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:39:39Z" name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw.18b15043b1be3b97 namespace: kserve-ci-e2e-test resourceVersion: "13254" uid: 9d78f8c0-4eb6-46f0-ba80-1576e354b698 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:39:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw namespace: kserve-ci-e2e-test resourceVersion: "12990" uid: 28616225-17ab-40a2-8c93-3f711b09517d kind: Event lastTimestamp: "2026-05-20T15:39:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:39:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:39:39Z" name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw.18b15043b233f73d namespace: kserve-ci-e2e-test resourceVersion: "13256" uid: 3f339eaa-11b9-4668-b0a7-7daf1901c7c1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:39:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw namespace: kserve-ci-e2e-test resourceVersion: "12990" uid: 28616225-17ab-40a2-8c93-3f711b09517d kind: Event lastTimestamp: "2026-05-20T15:39:39Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:e9391ee1281af7f81a916bc80b584691bef158df8f07d34a6fb179b3bd4767e7" metadata: creationTimestamp: "2026-05-20T15:39:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:39:39Z" name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw.18b15043b23ff1dc namespace: kserve-ci-e2e-test resourceVersion: "13257" uid: c5959022-3743-4bba-b6b5-9fe4ed877190 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:39:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw namespace: kserve-ci-e2e-test resourceVersion: "12990" uid: 28616225-17ab-40a2-8c93-3f711b09517d kind: Event lastTimestamp: "2026-05-20T15:39:41Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:e9391ee1281af7f81a916bc80b584691bef158df8f07d34a6fb179b3bd4767e7" in 2.521s (2.521s including waiting). Image size: 239688435 bytes.' metadata: creationTimestamp: "2026-05-20T15:39:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:39:41Z" name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw.18b15044488b54e3 namespace: kserve-ci-e2e-test resourceVersion: "13274" uid: 7a62d195-fcbf-4ead-8274-9f53b534e8c6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:39:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw namespace: kserve-ci-e2e-test resourceVersion: "12990" uid: 28616225-17ab-40a2-8c93-3f711b09517d kind: Event lastTimestamp: "2026-05-20T15:39:42Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-05-20T15:39:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:39:42Z" name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw.18b150444bc5e20e namespace: kserve-ci-e2e-test resourceVersion: "13276" uid: 52b4d038-5acf-408a-80ec-006871de362d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:39:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw namespace: kserve-ci-e2e-test resourceVersion: "12990" uid: 28616225-17ab-40a2-8c93-3f711b09517d kind: Event lastTimestamp: "2026-05-20T15:39:42Z" message: Started container agent metadata: creationTimestamp: "2026-05-20T15:39:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:39:42Z" name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw.18b150444c47fcf2 namespace: kserve-ci-e2e-test resourceVersion: "13277" uid: ddaeaeb0-4f42-4072-b0f6-b744b783e4f4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-05-20T15:39:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw namespace: kserve-ci-e2e-test resourceVersion: "12990" uid: 28616225-17ab-40a2-8c93-3f711b09517d kind: Event lastTimestamp: "2026-05-20T15:41:15Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:39:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:41:15Z" name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw.18b1504465c2a025 namespace: kserve-ci-e2e-test resourceVersion: "14105" uid: 2aa0212a-7693-4796-8707-c370a8102c37 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-20T15:39:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw namespace: kserve-ci-e2e-test resourceVersion: "12990" uid: 28616225-17ab-40a2-8c93-3f711b09517d kind: Event lastTimestamp: "2026-05-20T15:41:15Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T15:39:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:41:15Z" name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw.18b15044a183a979 namespace: kserve-ci-e2e-test resourceVersion: "14106" uid: 7dea7067-16d3-45a3-9e12-a3fe96fc4a35 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:40:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw namespace: kserve-ci-e2e-test resourceVersion: "12990" uid: 28616225-17ab-40a2-8c93-3f711b09517d kind: Event lastTimestamp: "2026-05-20T15:40:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T15:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:40:59Z" name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw.18b1505660599760 namespace: kserve-ci-e2e-test resourceVersion: "13877" uid: 206f10aa-8036-4e14-a489-a407ce7cf0c6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:40:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw namespace: kserve-ci-e2e-test resourceVersion: "12990" uid: 28616225-17ab-40a2-8c93-3f711b09517d kind: Event lastTimestamp: "2026-05-20T15:40:59Z" message: Stopping container agent metadata: creationTimestamp: "2026-05-20T15:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:40:59Z" name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw.18b150566059cebe namespace: kserve-ci-e2e-test resourceVersion: "13878" uid: e6960192-33e7-4aee-941d-a7b630158e7c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:40:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw namespace: kserve-ci-e2e-test resourceVersion: "12990" uid: 28616225-17ab-40a2-8c93-3f711b09517d kind: Event lastTimestamp: "2026-05-20T15:40:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:40:59Z" name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw.18b15056605a4b05 namespace: kserve-ci-e2e-test resourceVersion: "13881" uid: 60338042-d45d-407c-979e-e99ce7db6627 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:41:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw namespace: kserve-ci-e2e-test resourceVersion: "12990" uid: 28616225-17ab-40a2-8c93-3f711b09517d kind: Event lastTimestamp: "2026-05-20T15:41:14Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:41:14Z" name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw.18b150577db56368 namespace: kserve-ci-e2e-test resourceVersion: "14091" uid: 3d6991c4-a332-470b-abb7-0603c31a6794 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:39:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c namespace: kserve-ci-e2e-test resourceVersion: "12977" uid: 5047a753-bb8c-4d44-8f1d-6220a3409d63 kind: Event lastTimestamp: "2026-05-20T15:39:14Z" message: 'Created pod: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c-j4mxw' metadata: creationTimestamp: "2026-05-20T15:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:39:14Z" name: isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c.18b1503dec27b597 namespace: kserve-ci-e2e-test resourceVersion: "12989" uid: 9d17baa4-13a4-4194-abf9-50a750876c93 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:39:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-be7d1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12976" uid: f2f700e5-fb42-47bb-b142-e82ecd5cd5d7 kind: Event lastTimestamp: "2026-05-20T15:39:14Z" message: Scaled up replica set isvc-raw-sklearn-batcher-be7d1-predictor-d8667789c from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:39:14Z" name: isvc-raw-sklearn-batcher-be7d1-predictor.18b1503de569aeb1 namespace: kserve-ci-e2e-test resourceVersion: "12978" uid: d25c5a91-5920-4a6d-a094-8a286486238e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:39:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-be7d1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12986" uid: cc9dae9a-d0f8-4f3d-8053-f3af8a34148e kind: Event lastTimestamp: "2026-05-20T15:40:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:39:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:40:14Z" name: isvc-raw-sklearn-batcher-be7d1-predictor.18b150416b363c1c namespace: kserve-ci-e2e-test resourceVersion: "13522" uid: fac77259-77c3-4c59-b99f-5c6db134c79f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:39:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-be7d1-predictor namespace: kserve-ci-e2e-test resourceVersion: "12986" uid: cc9dae9a-d0f8-4f3d-8053-f3af8a34148e kind: Event lastTimestamp: "2026-05-20T15:40:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:39:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:40:14Z" name: isvc-raw-sklearn-batcher-be7d1-predictor.18b150416ba3b21e namespace: kserve-ci-e2e-test resourceVersion: "13523" uid: 4b998b68-9a7d-4b30-ad91-9e245127cedc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:40:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-be7d1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13159" uid: cc9dae9a-d0f8-4f3d-8053-f3af8a34148e kind: Event lastTimestamp: "2026-05-20T15:40:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:40:44Z" name: isvc-raw-sklearn-batcher-be7d1-predictor.18b1504f67cbcb37 namespace: kserve-ci-e2e-test resourceVersion: "13741" uid: b3c5133b-a6a6-4983-8e61-4daa4713d5f3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:40:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-be7d1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13159" uid: cc9dae9a-d0f8-4f3d-8053-f3af8a34148e kind: Event lastTimestamp: "2026-05-20T15:40:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:40:44Z" name: isvc-raw-sklearn-batcher-be7d1-predictor.18b1504f682f8b68 namespace: kserve-ci-e2e-test resourceVersion: "13742" uid: 9e58b789-4827-4825-85fd-5210cb4b1c0d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-05-20T15:39:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-be7d1 namespace: kserve-ci-e2e-test resourceVersion: "12966" uid: 2c040bcf-746d-4884-b2d6-9d758a5f2520 kind: Event lastTimestamp: "2026-05-20T15:40:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:40:45Z" name: isvc-raw-sklearn-batcher-be7d1.18b1503debdf5abb namespace: kserve-ci-e2e-test resourceVersion: "13756" uid: 33e86a16-0232-4d7b-8efb-c8989df8ee94 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:39:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-be7d1 namespace: kserve-ci-e2e-test resourceVersion: "12966" uid: 2c040bcf-746d-4884-b2d6-9d758a5f2520 kind: Event lastTimestamp: "2026-05-20T15:39:14Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-be7d1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-be7d1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:39:14Z" name: isvc-raw-sklearn-batcher-be7d1.18b1503dec45ed76 namespace: kserve-ci-e2e-test resourceVersion: "12992" uid: 66591357-98b6-4540-9635-b44d853bef3f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:39:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-be7d1 namespace: kserve-ci-e2e-test resourceVersion: "12966" uid: 2c040bcf-746d-4884-b2d6-9d758a5f2520 kind: Event lastTimestamp: "2026-05-20T15:39:14Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-be7d1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:39:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:39:14Z" name: isvc-raw-sklearn-batcher-be7d1.18b1503dec4653ce namespace: kserve-ci-e2e-test resourceVersion: "12994" uid: e92239b8-8499-47e5-a0e7-9792b4f68874 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:40:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-be7d1 namespace: kserve-ci-e2e-test resourceVersion: "13754" uid: 2c040bcf-746d-4884-b2d6-9d758a5f2520 kind: Event lastTimestamp: "2026-05-20T15:40:45Z" message: InferenceService [isvc-raw-sklearn-batcher-be7d1] is Ready metadata: creationTimestamp: "2026-05-20T15:40:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:40:45Z" name: isvc-raw-sklearn-batcher-be7d1.18b15053171c1908 namespace: kserve-ci-e2e-test resourceVersion: "13755" uid: a25e0c0e-7f3f-432f-8ae9-f4ee6f727dd2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:49:56.231484Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-c3463e-predictor-746949d687-t525f namespace: kserve-ci-e2e-test resourceVersion: "18616" uid: 882f2be3-ade1-4f59-a2db-a37617e8279d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-c3463e-predictor-746949d687-t525f to ip-10-0-140-154.ec2.internal metadata: creationTimestamp: "2026-05-20T15:49:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:49:56Z" name: isvc-secondary-c3463e-predictor-746949d687-t525f.18b150d34c00d5ca namespace: kserve-ci-e2e-test resourceVersion: "18620" uid: 9f8bed39-b05f-4e65-87ff-c781da9c7d4e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df74f7774-d6kd2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:49:56Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-c3463e-predictor-746949d687-t525f namespace: kserve-ci-e2e-test resourceVersion: "18618" uid: 882f2be3-ade1-4f59-a2db-a37617e8279d kind: Event lastTimestamp: "2026-05-20T15:49:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-secondary-c3463e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T15:49:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:49:56Z" name: isvc-secondary-c3463e-predictor-746949d687-t525f.18b150d35536796d namespace: kserve-ci-e2e-test resourceVersion: "18628" uid: 8ccc5ddb-f7a3-45de-8c53-667174a09c30 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:49:57Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-c3463e-predictor-746949d687-t525f namespace: kserve-ci-e2e-test resourceVersion: "18624" uid: 882f2be3-ade1-4f59-a2db-a37617e8279d kind: Event lastTimestamp: "2026-05-20T15:49:57Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:49:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:49:57Z" name: isvc-secondary-c3463e-predictor-746949d687-t525f.18b150d3899eaeb9 namespace: kserve-ci-e2e-test resourceVersion: "18637" uid: 0ee45fe9-6338-4e33-9736-44014ed95839 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:49:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-c3463e-predictor-746949d687-t525f namespace: kserve-ci-e2e-test resourceVersion: "18618" uid: 882f2be3-ade1-4f59-a2db-a37617e8279d kind: Event lastTimestamp: "2026-05-20T15:50:03Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:050b18baea3a969ddc2a15671f898a5f40ff53eb89f54c18856b130dc10bd894" already present on machine metadata: creationTimestamp: "2026-05-20T15:49:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:50:03Z" name: isvc-secondary-c3463e-predictor-746949d687-t525f.18b150d38be7065b namespace: kserve-ci-e2e-test resourceVersion: "18688" uid: 27c4f23b-c80e-43b3-abe1-6cd88416cd55 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:49:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-c3463e-predictor-746949d687-t525f namespace: kserve-ci-e2e-test resourceVersion: "18618" uid: 882f2be3-ade1-4f59-a2db-a37617e8279d kind: Event lastTimestamp: "2026-05-20T15:50:03Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T15:49:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:50:03Z" name: isvc-secondary-c3463e-predictor-746949d687-t525f.18b150d38f97a806 namespace: kserve-ci-e2e-test resourceVersion: "18691" uid: 75c3efc5-dc31-4a79-b050-84e4c579abd9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:49:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-c3463e-predictor-746949d687-t525f namespace: kserve-ci-e2e-test resourceVersion: "18618" uid: 882f2be3-ade1-4f59-a2db-a37617e8279d kind: Event lastTimestamp: "2026-05-20T15:50:03Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T15:49:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:50:03Z" name: isvc-secondary-c3463e-predictor-746949d687-t525f.18b150d390142761 namespace: kserve-ci-e2e-test resourceVersion: "18692" uid: 54283e1e-10a7-4800-b5fe-672f9e0954d3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:50:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-c3463e-predictor-746949d687-t525f namespace: kserve-ci-e2e-test resourceVersion: "18618" uid: 882f2be3-ade1-4f59-a2db-a37617e8279d kind: Event lastTimestamp: "2026-05-20T15:50:08Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-c3463e-predictor-746949d687-t525f_kserve-ci-e2e-test(882f2be3-ade1-4f59-a2db-a37617e8279d) metadata: creationTimestamp: "2026-05-20T15:50:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:50:08Z" name: isvc-secondary-c3463e-predictor-746949d687-t525f.18b150d637a47500 namespace: kserve-ci-e2e-test resourceVersion: "18732" uid: ae6a8cfe-e6c1-48ab-96d9-3bb4f8517e2f reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:49:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-c3463e-predictor-746949d687 namespace: kserve-ci-e2e-test resourceVersion: "18606" uid: b33558e2-81e8-43fc-ad1e-8b3b2b6503f6 kind: Event lastTimestamp: "2026-05-20T15:49:56Z" message: 'Created pod: isvc-secondary-c3463e-predictor-746949d687-t525f' metadata: creationTimestamp: "2026-05-20T15:49:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:49:56Z" name: isvc-secondary-c3463e-predictor-746949d687.18b150d34b9a3cbf namespace: kserve-ci-e2e-test resourceVersion: "18617" uid: 13ad01b0-c895-471d-b1bf-f30cfd0d6537 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:49:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-c3463e-predictor namespace: kserve-ci-e2e-test resourceVersion: "18604" uid: 8d119ef0-d45a-4abe-a4cc-740e72876127 kind: Event lastTimestamp: "2026-05-20T15:49:56Z" message: Scaled up replica set isvc-secondary-c3463e-predictor-746949d687 from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:49:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:49:56Z" name: isvc-secondary-c3463e-predictor.18b150d34825e203 namespace: kserve-ci-e2e-test resourceVersion: "18608" uid: f7c85c42-e543-40c5-8294-60c6d482da75 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:50:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-c3463e-predictor namespace: kserve-ci-e2e-test resourceVersion: "18605" uid: 74fe957d-2bb6-4c39-81dc-de21d1249b71 kind: Event lastTimestamp: "2026-05-20T15:50:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:50:11Z" name: isvc-secondary-c3463e-predictor.18b150d6c76a864c namespace: kserve-ci-e2e-test resourceVersion: "18755" uid: 36c32984-574c-46eb-a0eb-79f75e247e2c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:50:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-c3463e-predictor namespace: kserve-ci-e2e-test resourceVersion: "18605" uid: 74fe957d-2bb6-4c39-81dc-de21d1249b71 kind: Event lastTimestamp: "2026-05-20T15:50:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:50:11Z" name: isvc-secondary-c3463e-predictor.18b150d6c7d8c040 namespace: kserve-ci-e2e-test resourceVersion: "18757" uid: 068d3af4-d12d-495d-aecc-fc848a0a21c2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-05-20T15:49:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-c3463e namespace: kserve-ci-e2e-test resourceVersion: "18594" uid: ff33da02-b4d9-4ec7-bace-5cdab7d922b2 kind: Event lastTimestamp: "2026-05-20T15:50:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:49:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:50:10Z" name: isvc-secondary-c3463e.18b150d34801df3c namespace: kserve-ci-e2e-test resourceVersion: "18747" uid: 83f7db9a-cc10-45ed-a5ab-dfabc211ebb2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:49:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-c3463e namespace: kserve-ci-e2e-test resourceVersion: "18594" uid: ff33da02-b4d9-4ec7-bace-5cdab7d922b2 kind: Event lastTimestamp: "2026-05-20T15:49:56Z" message: 'Failed to update status for InferenceService "isvc-secondary-c3463e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-c3463e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:49:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:49:56Z" name: isvc-secondary-c3463e.18b150d34892b475 namespace: kserve-ci-e2e-test resourceVersion: "18610" uid: 8b758cc5-3851-4545-82ef-f2261f71246d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:49:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-c3463e namespace: kserve-ci-e2e-test resourceVersion: "18594" uid: ff33da02-b4d9-4ec7-bace-5cdab7d922b2 kind: Event lastTimestamp: "2026-05-20T15:49:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-c3463e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:49:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:49:56Z" name: isvc-secondary-c3463e.18b150d348931617 namespace: kserve-ci-e2e-test resourceVersion: "18612" uid: d85e5958-4efd-4138-8409-dacea326726e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:40:59.815315Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4 namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: b3579e22-6597-436b-a1ca-2c6fac6e14a9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4 to ip-10-0-140-154.ec2.internal metadata: creationTimestamp: "2026-05-20T15:40:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:40:59Z" name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4.18b15056671b9396 namespace: kserve-ci-e2e-test resourceVersion: "13902" uid: a58708b0-821f-4da8-a884-10a1a51cb664 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df74f7774-d6kd2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:41:00Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4 namespace: kserve-ci-e2e-test resourceVersion: "13914" uid: b3579e22-6597-436b-a1ca-2c6fac6e14a9 kind: Event lastTimestamp: "2026-05-20T15:41:00Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:41:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:41:00Z" name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4.18b1505681c6c764 namespace: kserve-ci-e2e-test resourceVersion: "13942" uid: 0ad152d6-5f1b-4aba-b5a4-ccbd8b10e84b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:41:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4 namespace: kserve-ci-e2e-test resourceVersion: "13899" uid: b3579e22-6597-436b-a1ca-2c6fac6e14a9 kind: Event lastTimestamp: "2026-05-20T15:41:00Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:050b18baea3a969ddc2a15671f898a5f40ff53eb89f54c18856b130dc10bd894" already present on machine metadata: creationTimestamp: "2026-05-20T15:41:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:41:00Z" name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4.18b1505683474085 namespace: kserve-ci-e2e-test resourceVersion: "13944" uid: 11861331-01c5-4dd9-b429-9d2baceed8e3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:41:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4 namespace: kserve-ci-e2e-test resourceVersion: "13899" uid: b3579e22-6597-436b-a1ca-2c6fac6e14a9 kind: Event lastTimestamp: "2026-05-20T15:41:00Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T15:41:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:41:00Z" name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4.18b1505686920dd7 namespace: kserve-ci-e2e-test resourceVersion: "13946" uid: 4c8d51d6-e3bf-499f-95ee-8bd377d9f7d9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:41:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4 namespace: kserve-ci-e2e-test resourceVersion: "13899" uid: b3579e22-6597-436b-a1ca-2c6fac6e14a9 kind: Event lastTimestamp: "2026-05-20T15:41:00Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T15:41:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:41:00Z" name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4.18b150568704e8c9 namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: adaae260-0081-42f5-9fce-4e3d8720dc3a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:41:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4 namespace: kserve-ci-e2e-test resourceVersion: "13899" uid: b3579e22-6597-436b-a1ca-2c6fac6e14a9 kind: Event lastTimestamp: "2026-05-20T15:41:04Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-05-20T15:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:41:04Z" name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4.18b150578fb7bfa4 namespace: kserve-ci-e2e-test resourceVersion: "14001" uid: d633831a-8c81-487e-a2e2-f1050bc629fe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:41:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4 namespace: kserve-ci-e2e-test resourceVersion: "13899" uid: b3579e22-6597-436b-a1ca-2c6fac6e14a9 kind: Event lastTimestamp: "2026-05-20T15:41:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:41:04Z" name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4.18b15057937a297d namespace: kserve-ci-e2e-test resourceVersion: "14006" uid: ba1d5c94-22a8-4382-adf3-9069aba7a793 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:41:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4 namespace: kserve-ci-e2e-test resourceVersion: "13899" uid: b3579e22-6597-436b-a1ca-2c6fac6e14a9 kind: Event lastTimestamp: "2026-05-20T15:41:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:41:04Z" name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4.18b1505793dd9c31 namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: 58de3e99-b59d-429e-ac2b-ec97e60fa593 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:41:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4 namespace: kserve-ci-e2e-test resourceVersion: "13899" uid: b3579e22-6597-436b-a1ca-2c6fac6e14a9 kind: Event lastTimestamp: "2026-05-20T15:41:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T15:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:41:04Z" name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4.18b1505793ebe11d namespace: kserve-ci-e2e-test resourceVersion: "14008" uid: a987bd28-3dc5-4dfc-9287-214f8b1b7c11 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:41:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4 namespace: kserve-ci-e2e-test resourceVersion: "13899" uid: b3579e22-6597-436b-a1ca-2c6fac6e14a9 kind: Event lastTimestamp: "2026-05-20T15:41:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:41:04Z" name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4.18b1505797000a44 namespace: kserve-ci-e2e-test resourceVersion: "14011" uid: e6453a92-3364-42aa-a402-7e40aa6cfd89 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:41:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4 namespace: kserve-ci-e2e-test resourceVersion: "13899" uid: b3579e22-6597-436b-a1ca-2c6fac6e14a9 kind: Event lastTimestamp: "2026-05-20T15:41:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:41:04Z" name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4.18b150579763de5b namespace: kserve-ci-e2e-test resourceVersion: "14012" uid: 63778fab-b066-422b-9d96-06e2fac603cb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-20T15:41:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4 namespace: kserve-ci-e2e-test resourceVersion: "13899" uid: b3579e22-6597-436b-a1ca-2c6fac6e14a9 kind: Event lastTimestamp: "2026-05-20T15:42:52Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:41:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:52Z" name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4.18b1505807a2349d namespace: kserve-ci-e2e-test resourceVersion: "15035" uid: 4fa1bca3-a9b7-42c0-9c4e-1804be702e85 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4 namespace: kserve-ci-e2e-test resourceVersion: "13899" uid: b3579e22-6597-436b-a1ca-2c6fac6e14a9 kind: Event lastTimestamp: "2026-05-20T15:42:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T15:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:50Z" name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4.18b150701a4897f5 namespace: kserve-ci-e2e-test resourceVersion: "14925" uid: 909374e6-9a74-49fe-8221-2426470f8a7e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4 namespace: kserve-ci-e2e-test resourceVersion: "13899" uid: b3579e22-6597-436b-a1ca-2c6fac6e14a9 kind: Event lastTimestamp: "2026-05-20T15:42:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:50Z" name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4.18b150701a49427d namespace: kserve-ci-e2e-test resourceVersion: "14926" uid: cbb9e360-3df2-4a27-9f65-c3a83732159b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:42:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4 namespace: kserve-ci-e2e-test resourceVersion: "13899" uid: b3579e22-6597-436b-a1ca-2c6fac6e14a9 kind: Event lastTimestamp: "2026-05-20T15:42:57Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:57Z" name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4.18b15070b5ebc707 namespace: kserve-ci-e2e-test resourceVersion: "15107" uid: 7d4a2491-db48-475e-a7f3-f2f4d3321019 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:40:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c namespace: kserve-ci-e2e-test resourceVersion: "13880" uid: 812b26a0-de4b-4ce5-ad05-9925d4529d92 kind: Event lastTimestamp: "2026-05-20T15:40:59Z" message: 'Created pod: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c-frrv4' metadata: creationTimestamp: "2026-05-20T15:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:40:59Z" name: isvc-sklearn-graph-raw-61d94-predictor-794dddc95c.18b1505666616c59 namespace: kserve-ci-e2e-test resourceVersion: "13897" uid: d9037d33-b677-48da-a495-a622f157f0ba reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:40:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-61d94-predictor namespace: kserve-ci-e2e-test resourceVersion: "13876" uid: 8db2abac-54d5-4d82-9f52-253a5d8d6aac kind: Event lastTimestamp: "2026-05-20T15:40:59Z" message: Scaled up replica set isvc-sklearn-graph-raw-61d94-predictor-794dddc95c from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:40:59Z" name: isvc-sklearn-graph-raw-61d94-predictor.18b15056615e4686 namespace: kserve-ci-e2e-test resourceVersion: "13883" uid: e524989b-bab5-4e95-b314-9a51434ac914 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:41:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-61d94-predictor namespace: kserve-ci-e2e-test resourceVersion: "13879" uid: 750fe8c7-6493-4700-bf67-37a6ec80f2d5 kind: Event lastTimestamp: "2026-05-20T15:41:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:41:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:41:44Z" name: isvc-sklearn-graph-raw-61d94-predictor.18b15059e06145bb namespace: kserve-ci-e2e-test resourceVersion: "14327" uid: 2d063abf-36b9-4644-8ef1-8ee367326607 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:41:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-61d94-predictor namespace: kserve-ci-e2e-test resourceVersion: "13879" uid: 750fe8c7-6493-4700-bf67-37a6ec80f2d5 kind: Event lastTimestamp: "2026-05-20T15:41:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:41:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:41:44Z" name: isvc-sklearn-graph-raw-61d94-predictor.18b15059e0cddbe6 namespace: kserve-ci-e2e-test resourceVersion: "14329" uid: e2ad00a9-baa6-4cd8-88f1-4001113a87f2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T15:41:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-61d94-predictor namespace: kserve-ci-e2e-test resourceVersion: "14096" uid: 750fe8c7-6493-4700-bf67-37a6ec80f2d5 kind: Event lastTimestamp: "2026-05-20T15:42:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:42:44Z" name: isvc-sklearn-graph-raw-61d94-predictor.18b1505d5fb02cae namespace: kserve-ci-e2e-test resourceVersion: "14821" uid: 846dd5a8-a4b9-4df8-b55b-3cb99d53af2f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T15:41:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-61d94-predictor namespace: kserve-ci-e2e-test resourceVersion: "14096" uid: 750fe8c7-6493-4700-bf67-37a6ec80f2d5 kind: Event lastTimestamp: "2026-05-20T15:42:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:41:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:42:44Z" name: isvc-sklearn-graph-raw-61d94-predictor.18b1505d601ace53 namespace: kserve-ci-e2e-test resourceVersion: "14822" uid: cfc2698d-9230-4587-a643-e9c745fc9e4a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-20T15:40:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-61d94 namespace: kserve-ci-e2e-test resourceVersion: "13850" uid: a0be0de0-bc88-4aff-a1df-724aa85925dd kind: Event lastTimestamp: "2026-05-20T15:42:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:42:22Z" name: isvc-sklearn-graph-raw-61d94.18b150566137d2b3 namespace: kserve-ci-e2e-test resourceVersion: "14617" uid: 61b27a32-1b97-40e7-8643-18cf0662d0ad reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:40:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-61d94 namespace: kserve-ci-e2e-test resourceVersion: "13850" uid: a0be0de0-bc88-4aff-a1df-724aa85925dd kind: Event lastTimestamp: "2026-05-20T15:40:59Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-61d94": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-61d94": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:40:59Z" name: isvc-sklearn-graph-raw-61d94.18b1505661ac1ceb namespace: kserve-ci-e2e-test resourceVersion: "13885" uid: 9933d5ee-fe97-4a2a-a220-f2a653f3cd9a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-61d94 namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: a0be0de0-bc88-4aff-a1df-724aa85925dd kind: Event lastTimestamp: "2026-05-20T15:42:22Z" message: InferenceService [isvc-sklearn-graph-raw-61d94] is Ready metadata: creationTimestamp: "2026-05-20T15:42:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:42:22Z" name: isvc-sklearn-graph-raw-61d94.18b15069c02c4a19 namespace: kserve-ci-e2e-test resourceVersion: "14614" uid: 621cdde3-7558-45f7-9aeb-dd995eba57aa reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:42:50.279509Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: bf1f60d1-cf8f-4598-982d-9faea05bb01a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s to ip-10-0-140-154.ec2.internal metadata: creationTimestamp: "2026-05-20T15:42:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:42:50Z" name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s.18b150701f495ae3 namespace: kserve-ci-e2e-test resourceVersion: "14945" uid: fdf31ccc-4f8f-4552-a662-491c5d28dc4b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df74f7774-d6kd2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:50Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: bf1f60d1-cf8f-4598-982d-9faea05bb01a kind: Event lastTimestamp: "2026-05-20T15:42:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-raw-hpa-86fe3-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T15:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:50Z" name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s.18b150702c450e39 namespace: kserve-ci-e2e-test resourceVersion: "14987" uid: 3bc3b9d1-570f-4d65-a8b7-88cfa5a06432 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:51Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s namespace: kserve-ci-e2e-test resourceVersion: "14952" uid: bf1f60d1-cf8f-4598-982d-9faea05bb01a kind: Event lastTimestamp: "2026-05-20T15:42:51Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:42:51Z" name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s.18b150705ce86ad2 namespace: kserve-ci-e2e-test resourceVersion: "15010" uid: 31b420d8-c645-4164-93eb-98b53e4931ca reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: bf1f60d1-cf8f-4598-982d-9faea05bb01a kind: Event lastTimestamp: "2026-05-20T15:42:51Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:050b18baea3a969ddc2a15671f898a5f40ff53eb89f54c18856b130dc10bd894" already present on machine metadata: creationTimestamp: "2026-05-20T15:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:51Z" name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s.18b150705e7d4405 namespace: kserve-ci-e2e-test resourceVersion: "15012" uid: dac4bb16-7882-4810-a9d6-bde4a3f2eba5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: bf1f60d1-cf8f-4598-982d-9faea05bb01a kind: Event lastTimestamp: "2026-05-20T15:42:51Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T15:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:51Z" name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s.18b1507061ea919a namespace: kserve-ci-e2e-test resourceVersion: "15016" uid: 749869ab-5c49-4664-9008-bee0d57067d7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: bf1f60d1-cf8f-4598-982d-9faea05bb01a kind: Event lastTimestamp: "2026-05-20T15:42:51Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T15:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:51Z" name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s.18b1507062694a01 namespace: kserve-ci-e2e-test resourceVersion: "15017" uid: c87aab21-bbd8-424a-891f-f6743a6c2f9f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: bf1f60d1-cf8f-4598-982d-9faea05bb01a kind: Event lastTimestamp: "2026-05-20T15:42:55Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-05-20T15:42:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:55Z" name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s.18b15071458fd326 namespace: kserve-ci-e2e-test resourceVersion: "15066" uid: b30bac08-6bca-4c76-bef2-44196f6080a3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: bf1f60d1-cf8f-4598-982d-9faea05bb01a kind: Event lastTimestamp: "2026-05-20T15:42:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:42:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:55Z" name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s.18b1507149d22c9c namespace: kserve-ci-e2e-test resourceVersion: "15070" uid: 59580b13-c18f-40dc-ab95-5367d0587b47 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: bf1f60d1-cf8f-4598-982d-9faea05bb01a kind: Event lastTimestamp: "2026-05-20T15:42:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:42:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:55Z" name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s.18b150714a632f68 namespace: kserve-ci-e2e-test resourceVersion: "15074" uid: c168285d-31e3-49de-8afc-0956cf87aae4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: bf1f60d1-cf8f-4598-982d-9faea05bb01a kind: Event lastTimestamp: "2026-05-20T15:42:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T15:42:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:55Z" name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s.18b150714a700c5b namespace: kserve-ci-e2e-test resourceVersion: "15076" uid: 5276bd35-feee-42e2-9b99-a71705fa08ea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: bf1f60d1-cf8f-4598-982d-9faea05bb01a kind: Event lastTimestamp: "2026-05-20T15:42:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:42:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:55Z" name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s.18b150714de534a0 namespace: kserve-ci-e2e-test resourceVersion: "15081" uid: 07209ed5-7f3a-4624-8a2f-559c42a8e961 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: bf1f60d1-cf8f-4598-982d-9faea05bb01a kind: Event lastTimestamp: "2026-05-20T15:42:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:42:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:55Z" name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s.18b150714e5702d3 namespace: kserve-ci-e2e-test resourceVersion: "15084" uid: 1bb99304-bdb8-4404-8bfb-c00f680ec263 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-20T15:42:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: bf1f60d1-cf8f-4598-982d-9faea05bb01a kind: Event lastTimestamp: "2026-05-20T15:44:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:42:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:32Z" name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s.18b1507181a79967 namespace: kserve-ci-e2e-test resourceVersion: "15953" uid: 8bfef3f2-e593-426b-b62f-4897748cc476 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: bf1f60d1-cf8f-4598-982d-9faea05bb01a kind: Event lastTimestamp: "2026-05-20T15:44:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T15:44:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:30Z" name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s.18b1508773e088d8 namespace: kserve-ci-e2e-test resourceVersion: "15895" uid: 0bd4ea46-e40c-4593-a4b8-6c7dc5c5e7e9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: bf1f60d1-cf8f-4598-982d-9faea05bb01a kind: Event lastTimestamp: "2026-05-20T15:44:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:44:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:30Z" name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s.18b1508773e12d8a namespace: kserve-ci-e2e-test resourceVersion: "15897" uid: 1026f918-b340-42ec-a574-ccee556902bc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: bf1f60d1-cf8f-4598-982d-9faea05bb01a kind: Event lastTimestamp: "2026-05-20T15:44:32Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:32Z" name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s.18b15087dbfd5669 namespace: kserve-ci-e2e-test resourceVersion: "15952" uid: 13e394ed-cdf4-4b6f-a312-f7b0eb630574 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849 namespace: kserve-ci-e2e-test resourceVersion: "14907" uid: c346437d-21e3-4d59-abba-b07eb0f555e4 kind: Event lastTimestamp: "2026-05-20T15:42:50Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849-j6z8s' metadata: creationTimestamp: "2026-05-20T15:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:42:50Z" name: isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849.18b150701e765b5e namespace: kserve-ci-e2e-test resourceVersion: "14939" uid: cc998ad6-a2ce-48ec-aa87-2aa70f284564 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-86fe3-predictor namespace: kserve-ci-e2e-test resourceVersion: "14902" uid: b1ec5745-53f8-42e1-87d6-9472d5bb8aaf kind: Event lastTimestamp: "2026-05-20T15:42:50Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-86fe3-predictor-8f4dc8849 from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:42:50Z" name: isvc-sklearn-graph-raw-hpa-86fe3-predictor.18b15070178d7f35 namespace: kserve-ci-e2e-test resourceVersion: "14911" uid: 2a2b167c-b988-4a92-86a0-afaa25cd50c3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:43:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-86fe3-predictor namespace: kserve-ci-e2e-test resourceVersion: "14904" uid: bff8ccc1-8d54-4796-a8c0-bb09453e32c3 kind: Event lastTimestamp: "2026-05-20T15:43:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:43:20Z" name: isvc-sklearn-graph-raw-hpa-86fe3-predictor.18b1507395f763dc namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: 2f2b563a-63f3-4fa6-ac23-73ee980623e6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:43:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-86fe3-predictor namespace: kserve-ci-e2e-test resourceVersion: "14904" uid: bff8ccc1-8d54-4796-a8c0-bb09453e32c3 kind: Event lastTimestamp: "2026-05-20T15:43:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:43:20Z" name: isvc-sklearn-graph-raw-hpa-86fe3-predictor.18b1507396c45c8f namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: 852ec819-2827-4495-b167-64662370672a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:43:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-86fe3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15165" uid: bff8ccc1-8d54-4796-a8c0-bb09453e32c3 kind: Event lastTimestamp: "2026-05-20T15:44:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:43:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:20Z" name: isvc-sklearn-graph-raw-hpa-86fe3-predictor.18b1507a956e1a51 namespace: kserve-ci-e2e-test resourceVersion: "15753" uid: f361457c-cfa1-4823-8f04-dcb29a174fc2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:43:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-86fe3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15165" uid: bff8ccc1-8d54-4796-a8c0-bb09453e32c3 kind: Event lastTimestamp: "2026-05-20T15:44:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:43:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:20Z" name: isvc-sklearn-graph-raw-hpa-86fe3-predictor.18b1507a96203a5a namespace: kserve-ci-e2e-test resourceVersion: "15754" uid: 9bc08088-11d3-43a7-9c0c-82733ffe6799 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-05-20T15:42:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-86fe3 namespace: kserve-ci-e2e-test resourceVersion: "14886" uid: 860b3bd6-4033-4384-ae20-7d90bce54ec6 kind: Event lastTimestamp: "2026-05-20T15:44:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:44:02Z" name: isvc-sklearn-graph-raw-hpa-86fe3.18b15070170a8228 namespace: kserve-ci-e2e-test resourceVersion: "15592" uid: 900cc066-e89a-4259-9a72-a0baf6b1c99d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-86fe3 namespace: kserve-ci-e2e-test resourceVersion: "14886" uid: 860b3bd6-4033-4384-ae20-7d90bce54ec6 kind: Event lastTimestamp: "2026-05-20T15:42:50Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-86fe3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-86fe3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:42:50Z" name: isvc-sklearn-graph-raw-hpa-86fe3.18b1507017c769fa namespace: kserve-ci-e2e-test resourceVersion: "14914" uid: 7636a7ba-2fe4-4f51-a9a9-fd6f3fc5d909 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-86fe3 namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: 860b3bd6-4033-4384-ae20-7d90bce54ec6 kind: Event lastTimestamp: "2026-05-20T15:44:02Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-86fe3] is Ready metadata: creationTimestamp: "2026-05-20T15:44:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:44:02Z" name: isvc-sklearn-graph-raw-hpa-86fe3.18b15080e67a81ee namespace: kserve-ci-e2e-test resourceVersion: "15591" uid: 5725b041-83b6-48ae-993c-6fbc76af7ab6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:46:05.800906Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz namespace: kserve-ci-e2e-test resourceVersion: "16779" uid: 902200ca-05c3-417b-85ce-5799d3e757f4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz to ip-10-0-140-154.ec2.internal metadata: creationTimestamp: "2026-05-20T15:46:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:46:05Z" name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz.18b1509da54523f1 namespace: kserve-ci-e2e-test resourceVersion: "16784" uid: aae4a524-693a-49eb-80a2-769c7c568843 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df74f7774-d6kd2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:06Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz namespace: kserve-ci-e2e-test resourceVersion: "16789" uid: 902200ca-05c3-417b-85ce-5799d3e757f4 kind: Event lastTimestamp: "2026-05-20T15:46:06Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:46:06Z" name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz.18b1509dbf7d14c9 namespace: kserve-ci-e2e-test resourceVersion: "16813" uid: fc23cb4b-ad69-4484-8a5a-5fd81cef0575 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: 902200ca-05c3-417b-85ce-5799d3e757f4 kind: Event lastTimestamp: "2026-05-20T15:46:06Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:050b18baea3a969ddc2a15671f898a5f40ff53eb89f54c18856b130dc10bd894" already present on machine metadata: creationTimestamp: "2026-05-20T15:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:46:06Z" name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz.18b1509dc0e2efa1 namespace: kserve-ci-e2e-test resourceVersion: "16815" uid: e9a1cdf2-42b6-430b-b74b-e01b3dc1ec1e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: 902200ca-05c3-417b-85ce-5799d3e757f4 kind: Event lastTimestamp: "2026-05-20T15:46:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T15:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:46:06Z" name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz.18b1509dc3c217f8 namespace: kserve-ci-e2e-test resourceVersion: "16816" uid: 38cea6c1-1095-4591-88d9-c8eb2a9ed673 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: 902200ca-05c3-417b-85ce-5799d3e757f4 kind: Event lastTimestamp: "2026-05-20T15:46:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T15:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:46:06Z" name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz.18b1509dc44cd68b namespace: kserve-ci-e2e-test resourceVersion: "16817" uid: b4560426-a4fd-42fe-8ec9-5c56dc9d26b6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: 902200ca-05c3-417b-85ce-5799d3e757f4 kind: Event lastTimestamp: "2026-05-20T15:46:10Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-05-20T15:46:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:46:10Z" name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz.18b1509ed737f812 namespace: kserve-ci-e2e-test resourceVersion: "16856" uid: b49c4ad9-98a0-40d5-85ee-e5e03c0c57fb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: 902200ca-05c3-417b-85ce-5799d3e757f4 kind: Event lastTimestamp: "2026-05-20T15:46:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:46:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:46:10Z" name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz.18b1509eda7bf348 namespace: kserve-ci-e2e-test resourceVersion: "16860" uid: 7edd7c19-6dfe-4111-9a04-1ecf7c107b1a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: 902200ca-05c3-417b-85ce-5799d3e757f4 kind: Event lastTimestamp: "2026-05-20T15:46:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:46:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:46:10Z" name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz.18b1509edaed7b2d namespace: kserve-ci-e2e-test resourceVersion: "16862" uid: 2fab5a4f-7e9f-4e2f-91b7-0732759cbad7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: 902200ca-05c3-417b-85ce-5799d3e757f4 kind: Event lastTimestamp: "2026-05-20T15:46:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T15:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:46:11Z" name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz.18b1509edaf92d38 namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: 7170e996-4749-443a-82e1-4c8ad8b4a432 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: 902200ca-05c3-417b-85ce-5799d3e757f4 kind: Event lastTimestamp: "2026-05-20T15:46:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:46:11Z" name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz.18b1509ede2a60b2 namespace: kserve-ci-e2e-test resourceVersion: "16866" uid: bc9e8ff3-4323-4d0f-8065-72b0d4ca8ed8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: 902200ca-05c3-417b-85ce-5799d3e757f4 kind: Event lastTimestamp: "2026-05-20T15:46:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:46:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:46:11Z" name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz.18b1509edeaee65d namespace: kserve-ci-e2e-test resourceVersion: "16867" uid: 4bdfd863-71cc-457a-93a2-519acf057414 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 17 eventTime: null firstTimestamp: "2026-05-20T15:46:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: 902200ca-05c3-417b-85ce-5799d3e757f4 kind: Event lastTimestamp: "2026-05-20T15:48:43Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:46:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:48:43Z" name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz.18b1509f4f0e9b02 namespace: kserve-ci-e2e-test resourceVersion: "18054" uid: 7d691f45-4cad-4270-817b-635a403bf08c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:48:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: 902200ca-05c3-417b-85ce-5799d3e757f4 kind: Event lastTimestamp: "2026-05-20T15:48:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T15:48:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:48:35Z" name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz.18b150c09d5bc582 namespace: kserve-ci-e2e-test resourceVersion: "17948" uid: d2716fd6-7208-41cb-9c96-df9cdd81cbd6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:48:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: 902200ca-05c3-417b-85ce-5799d3e757f4 kind: Event lastTimestamp: "2026-05-20T15:48:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:48:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:48:35Z" name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz.18b150c09d5c8e0a namespace: kserve-ci-e2e-test resourceVersion: "17950" uid: 8a9badeb-fb33-434d-b67d-0c377bbbab9e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:48:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: 902200ca-05c3-417b-85ce-5799d3e757f4 kind: Event lastTimestamp: "2026-05-20T15:48:43Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:48:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:48:43Z" name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz.18b150c14d879109 namespace: kserve-ci-e2e-test resourceVersion: "18058" uid: 8981b2b7-5ed1-4dfe-bd1d-e0544b980b2b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9 namespace: kserve-ci-e2e-test resourceVersion: "16755" uid: b7eb0446-fa24-47bf-9c18-bca8d987dfc6 kind: Event lastTimestamp: "2026-05-20T15:46:05Z" message: 'Created pod: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9-pltfz' metadata: creationTimestamp: "2026-05-20T15:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:46:05Z" name: isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9.18b1509da4db9682 namespace: kserve-ci-e2e-test resourceVersion: "16781" uid: 1f4a179b-96f6-4c22-ba48-094f3fc67407 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-db50d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16750" uid: e87a9a44-1cce-4cf3-830e-528e12fd0e17 kind: Event lastTimestamp: "2026-05-20T15:46:05Z" message: Scaled up replica set isvc-sklearn-scale-raw-db50d-predictor-8f6c55c9 from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:46:05Z" name: isvc-sklearn-scale-raw-db50d-predictor.18b1509da1213f8a namespace: kserve-ci-e2e-test resourceVersion: "16761" uid: f77cf956-fed6-4365-a6d5-d36451d51582 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:46:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-db50d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16754" uid: 8a534bd7-7ef2-4f7c-9d68-6779ca901465 kind: Event lastTimestamp: "2026-05-20T15:46:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:46:35Z" name: isvc-sklearn-scale-raw-db50d-predictor.18b150a12040dc79 namespace: kserve-ci-e2e-test resourceVersion: "17057" uid: b0139b33-4cd8-4b03-94af-9ac9b76f3fe3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:46:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-db50d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16754" uid: 8a534bd7-7ef2-4f7c-9d68-6779ca901465 kind: Event lastTimestamp: "2026-05-20T15:46:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:46:35Z" name: isvc-sklearn-scale-raw-db50d-predictor.18b150a120a6a4c2 namespace: kserve-ci-e2e-test resourceVersion: "17058" uid: bdc8c0ed-b5d3-439b-aa36-cf052e0c39c0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-20T15:46:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-db50d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16947" uid: 8a534bd7-7ef2-4f7c-9d68-6779ca901465 kind: Event lastTimestamp: "2026-05-20T15:48:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:48:35Z" name: isvc-sklearn-scale-raw-db50d-predictor.18b150a81ed58847 namespace: kserve-ci-e2e-test resourceVersion: "17926" uid: 08837c42-dc48-41ad-9313-6fb3a3e41587 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-20T15:46:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-db50d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16947" uid: 8a534bd7-7ef2-4f7c-9d68-6779ca901465 kind: Event lastTimestamp: "2026-05-20T15:48:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:48:35Z" name: isvc-sklearn-scale-raw-db50d-predictor.18b150a81f479854 namespace: kserve-ci-e2e-test resourceVersion: "17927" uid: 42c8dfdb-45e3-4630-ba81-4a60735a05aa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-05-20T15:46:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-db50d namespace: kserve-ci-e2e-test resourceVersion: "16737" uid: 2757488d-f946-480c-8ae8-467dce58140a kind: Event lastTimestamp: "2026-05-20T15:48:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:48:33Z" name: isvc-sklearn-scale-raw-db50d.18b1509da12a104b namespace: kserve-ci-e2e-test resourceVersion: "17906" uid: ae82e795-9fa6-4868-8efc-8eeb29f5785e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-db50d namespace: kserve-ci-e2e-test resourceVersion: "16737" uid: 2757488d-f946-480c-8ae8-467dce58140a kind: Event lastTimestamp: "2026-05-20T15:46:05Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-db50d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-db50d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:46:05Z" name: isvc-sklearn-scale-raw-db50d.18b1509da1be6f53 namespace: kserve-ci-e2e-test resourceVersion: "16767" uid: a9814416-ccf3-43ea-9d69-6c40083f3ed6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:48:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-db50d namespace: kserve-ci-e2e-test resourceVersion: "17904" uid: 2757488d-f946-480c-8ae8-467dce58140a kind: Event lastTimestamp: "2026-05-20T15:48:33Z" message: InferenceService [isvc-sklearn-scale-raw-db50d] is Ready metadata: creationTimestamp: "2026-05-20T15:48:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:48:33Z" name: isvc-sklearn-scale-raw-db50d.18b150c01ef6e8e2 namespace: kserve-ci-e2e-test resourceVersion: "17905" uid: d22a3b9c-2caa-40e2-a7ad-17b384ebb815 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:40:59.900202Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq namespace: kserve-ci-e2e-test resourceVersion: "13921" uid: 0cbbf461-dd50-48d3-acc8-57d1c14b4c65 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq to ip-10-0-140-154.ec2.internal metadata: creationTimestamp: "2026-05-20T15:40:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:40:59Z" name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq.18b150566c2acc5c namespace: kserve-ci-e2e-test resourceVersion: "13925" uid: 90807219-7d9d-406b-9793-73061b8791c8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df74f7774-d6kd2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:41:00Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq namespace: kserve-ci-e2e-test resourceVersion: "13933" uid: 0cbbf461-dd50-48d3-acc8-57d1c14b4c65 kind: Event lastTimestamp: "2026-05-20T15:41:00Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:41:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:41:00Z" name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq.18b150568667f8db namespace: kserve-ci-e2e-test resourceVersion: "13945" uid: 7127568b-a2e6-466a-aad2-da70432bcaa3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:41:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 0cbbf461-dd50-48d3-acc8-57d1c14b4c65 kind: Event lastTimestamp: "2026-05-20T15:41:00Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:050b18baea3a969ddc2a15671f898a5f40ff53eb89f54c18856b130dc10bd894" already present on machine metadata: creationTimestamp: "2026-05-20T15:41:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:41:00Z" name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq.18b1505688255aaf namespace: kserve-ci-e2e-test resourceVersion: "13949" uid: 54bb18b8-7d44-43b2-8e1a-ae87a74f1cff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:41:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 0cbbf461-dd50-48d3-acc8-57d1c14b4c65 kind: Event lastTimestamp: "2026-05-20T15:41:00Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T15:41:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:41:00Z" name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq.18b150568b6c81b5 namespace: kserve-ci-e2e-test resourceVersion: "13950" uid: a2a9941c-ae25-4cfb-9b08-d18379237f31 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:41:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 0cbbf461-dd50-48d3-acc8-57d1c14b4c65 kind: Event lastTimestamp: "2026-05-20T15:41:00Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T15:41:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:41:00Z" name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq.18b150568bdd146d namespace: kserve-ci-e2e-test resourceVersion: "13951" uid: 4ceb8747-beb9-4617-bf35-646cdf8bf7d4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:41:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 0cbbf461-dd50-48d3-acc8-57d1c14b4c65 kind: Event lastTimestamp: "2026-05-20T15:41:04Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-05-20T15:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:41:04Z" name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq.18b150578f82f476 namespace: kserve-ci-e2e-test resourceVersion: "14000" uid: 91fce23f-ee94-4e59-9d5d-47c2f0494869 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:41:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 0cbbf461-dd50-48d3-acc8-57d1c14b4c65 kind: Event lastTimestamp: "2026-05-20T15:41:25Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 20.975s (20.975s including waiting). Image size: 1306444539 bytes.' metadata: creationTimestamp: "2026-05-20T15:41:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:41:25Z" name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq.18b1505c71c3cf30 namespace: kserve-ci-e2e-test resourceVersion: "14174" uid: f5c51f75-d55f-45f2-a89f-b8a139358376 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:41:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 0cbbf461-dd50-48d3-acc8-57d1c14b4c65 kind: Event lastTimestamp: "2026-05-20T15:41:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:41:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:41:25Z" name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq.18b1505c74e45dc5 namespace: kserve-ci-e2e-test resourceVersion: "14176" uid: 98bf73d1-bd75-4218-a113-fed8a6ffb465 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:41:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 0cbbf461-dd50-48d3-acc8-57d1c14b4c65 kind: Event lastTimestamp: "2026-05-20T15:41:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:41:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:41:25Z" name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq.18b1505c754d6426 namespace: kserve-ci-e2e-test resourceVersion: "14177" uid: a3f64ca2-86a0-4f69-9376-342fe1fe9a50 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:41:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 0cbbf461-dd50-48d3-acc8-57d1c14b4c65 kind: Event lastTimestamp: "2026-05-20T15:41:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T15:41:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:41:25Z" name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq.18b1505c755a45ca namespace: kserve-ci-e2e-test resourceVersion: "14178" uid: 4b2753ed-3e6e-427d-9e4d-4cf00d9f1818 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:41:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 0cbbf461-dd50-48d3-acc8-57d1c14b4c65 kind: Event lastTimestamp: "2026-05-20T15:41:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:41:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:41:25Z" name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq.18b1505c78bbec65 namespace: kserve-ci-e2e-test resourceVersion: "14179" uid: 5cb12a87-c613-4e3d-8573-52a02644d1c2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:41:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 0cbbf461-dd50-48d3-acc8-57d1c14b4c65 kind: Event lastTimestamp: "2026-05-20T15:41:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:41:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:41:25Z" name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq.18b1505c792dc210 namespace: kserve-ci-e2e-test resourceVersion: "14180" uid: d187975f-af6e-467a-9cee-5938af85f834 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-20T15:41:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 0cbbf461-dd50-48d3-acc8-57d1c14b4c65 kind: Event lastTimestamp: "2026-05-20T15:42:53Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:41:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:53Z" name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq.18b1505cf05fee09 namespace: kserve-ci-e2e-test resourceVersion: "15048" uid: b0ee2c4c-9c1f-471a-bfde-5ee38a716724 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 0cbbf461-dd50-48d3-acc8-57d1c14b4c65 kind: Event lastTimestamp: "2026-05-20T15:42:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T15:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:50Z" name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq.18b1507027b1e7eb namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 8718f416-47fc-4ba0-a439-676ecb9626b2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 0cbbf461-dd50-48d3-acc8-57d1c14b4c65 kind: Event lastTimestamp: "2026-05-20T15:42:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:50Z" name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq.18b1507027b2983d namespace: kserve-ci-e2e-test resourceVersion: "14984" uid: 13972f99-89b6-4219-9a6f-528889a591e8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: 0cbbf461-dd50-48d3-acc8-57d1c14b4c65 kind: Event lastTimestamp: "2026-05-20T15:42:53Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:42:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:53Z" name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq.18b15070f68e9d75 namespace: kserve-ci-e2e-test resourceVersion: "15047" uid: f7f675fb-92b5-4da7-99f0-bf4115d05db7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:40:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c namespace: kserve-ci-e2e-test resourceVersion: "13909" uid: 28288906-dbbc-4d93-8966-6247a73fcadb kind: Event lastTimestamp: "2026-05-20T15:40:59Z" message: 'Created pod: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c-4pgkq' metadata: creationTimestamp: "2026-05-20T15:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:40:59Z" name: isvc-xgboost-graph-raw-61d94-predictor-66c75f745c.18b150566bb6313d namespace: kserve-ci-e2e-test resourceVersion: "13922" uid: 2a682898-45b9-4eea-b522-27fdf344afa4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:40:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-61d94-predictor namespace: kserve-ci-e2e-test resourceVersion: "13907" uid: 8ebada94-8289-4191-bc28-c7196349281e kind: Event lastTimestamp: "2026-05-20T15:40:59Z" message: Scaled up replica set isvc-xgboost-graph-raw-61d94-predictor-66c75f745c from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:40:59Z" name: isvc-xgboost-graph-raw-61d94-predictor.18b1505668df7799 namespace: kserve-ci-e2e-test resourceVersion: "13912" uid: 950a88a8-9e3a-4024-be00-13fd02fb7067 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:41:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-61d94-predictor namespace: kserve-ci-e2e-test resourceVersion: "13908" uid: 1e0ea3da-1593-4b04-bd72-463ecc63154e kind: Event lastTimestamp: "2026-05-20T15:41:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:41:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:41:59Z" name: isvc-xgboost-graph-raw-61d94-predictor.18b15059e79d73a4 namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: 0e16261d-4371-4e09-9a1c-716091c20c82 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:41:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-61d94-predictor namespace: kserve-ci-e2e-test resourceVersion: "13908" uid: 1e0ea3da-1593-4b04-bd72-463ecc63154e kind: Event lastTimestamp: "2026-05-20T15:41:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:41:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:41:59Z" name: isvc-xgboost-graph-raw-61d94-predictor.18b15059e80a4241 namespace: kserve-ci-e2e-test resourceVersion: "14441" uid: ec84a351-bf3a-4c92-84af-2cf1d7a3146c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:42:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-61d94-predictor namespace: kserve-ci-e2e-test resourceVersion: "14100" uid: 1e0ea3da-1593-4b04-bd72-463ecc63154e kind: Event lastTimestamp: "2026-05-20T15:42:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:42:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:42:44Z" name: isvc-xgboost-graph-raw-61d94-predictor.18b15067e3391af8 namespace: kserve-ci-e2e-test resourceVersion: "14823" uid: eb13bdd8-5a7f-402f-a942-e94512d85705 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:42:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-61d94-predictor namespace: kserve-ci-e2e-test resourceVersion: "14100" uid: 1e0ea3da-1593-4b04-bd72-463ecc63154e kind: Event lastTimestamp: "2026-05-20T15:42:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:42:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:42:44Z" name: isvc-xgboost-graph-raw-61d94-predictor.18b15067e3a73d5d namespace: kserve-ci-e2e-test resourceVersion: "14824" uid: 035ab527-09e5-4ce3-ab43-44aa2efed922 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-20T15:40:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-61d94 namespace: kserve-ci-e2e-test resourceVersion: "13890" uid: 9b3f5a13-9d2f-4e45-b1bc-a3cf2ad81cdf kind: Event lastTimestamp: "2026-05-20T15:42:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:42:34Z" name: isvc-xgboost-graph-raw-61d94.18b1505668deff19 namespace: kserve-ci-e2e-test resourceVersion: "14710" uid: bc0bf42a-7f23-499a-92a7-ca80b7bdc2e8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:40:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-61d94 namespace: kserve-ci-e2e-test resourceVersion: "13890" uid: 9b3f5a13-9d2f-4e45-b1bc-a3cf2ad81cdf kind: Event lastTimestamp: "2026-05-20T15:40:59Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-61d94": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-61d94": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:40:59Z" name: isvc-xgboost-graph-raw-61d94.18b1505669b032db namespace: kserve-ci-e2e-test resourceVersion: "13917" uid: dacbc052-8c96-4b9c-859d-a7a07de8c2b8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-61d94 namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: 9b3f5a13-9d2f-4e45-b1bc-a3cf2ad81cdf kind: Event lastTimestamp: "2026-05-20T15:42:33Z" message: InferenceService [isvc-xgboost-graph-raw-61d94] is Ready metadata: creationTimestamp: "2026-05-20T15:42:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:42:33Z" name: isvc-xgboost-graph-raw-61d94.18b1506c54575ca5 namespace: kserve-ci-e2e-test resourceVersion: "14709" uid: f6587d7a-9eb7-48a5-8512-fcfb6a618711 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:42:50.357191Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz namespace: kserve-ci-e2e-test resourceVersion: "14955" uid: 0b36bd82-8eeb-454e-9ef7-6d85bc801e0a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz to ip-10-0-140-154.ec2.internal metadata: creationTimestamp: "2026-05-20T15:42:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:42:50Z" name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz.18b1507023eaa95c namespace: kserve-ci-e2e-test resourceVersion: "14961" uid: 333b8b89-1241-486f-a3cb-4d969a352487 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df74f7774-d6kd2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:50Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz namespace: kserve-ci-e2e-test resourceVersion: "14976" uid: 0b36bd82-8eeb-454e-9ef7-6d85bc801e0a kind: Event lastTimestamp: "2026-05-20T15:42:50Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:42:50Z" name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz.18b150703e15b92f namespace: kserve-ci-e2e-test resourceVersion: "14995" uid: 28ac7038-f51b-4688-8a5b-6643c551e59a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz namespace: kserve-ci-e2e-test resourceVersion: "14958" uid: 0b36bd82-8eeb-454e-9ef7-6d85bc801e0a kind: Event lastTimestamp: "2026-05-20T15:42:50Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:050b18baea3a969ddc2a15671f898a5f40ff53eb89f54c18856b130dc10bd894" already present on machine metadata: creationTimestamp: "2026-05-20T15:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:50Z" name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz.18b150704078e407 namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: c4d8720e-7e72-433a-bb41-84ccb0e715a9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz namespace: kserve-ci-e2e-test resourceVersion: "14958" uid: 0b36bd82-8eeb-454e-9ef7-6d85bc801e0a kind: Event lastTimestamp: "2026-05-20T15:42:50Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T15:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:50Z" name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz.18b15070436ecb70 namespace: kserve-ci-e2e-test resourceVersion: "14999" uid: 39711b5b-f356-4217-8382-3700c2487cb2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz namespace: kserve-ci-e2e-test resourceVersion: "14958" uid: 0b36bd82-8eeb-454e-9ef7-6d85bc801e0a kind: Event lastTimestamp: "2026-05-20T15:42:50Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T15:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:50Z" name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz.18b150704421b46e namespace: kserve-ci-e2e-test resourceVersion: "15000" uid: a7544578-a200-405a-980b-fa7602003f62 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz namespace: kserve-ci-e2e-test resourceVersion: "14958" uid: 0b36bd82-8eeb-454e-9ef7-6d85bc801e0a kind: Event lastTimestamp: "2026-05-20T15:42:55Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-05-20T15:42:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:55Z" name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz.18b1507145bd8b9d namespace: kserve-ci-e2e-test resourceVersion: "15067" uid: 86fbef1c-7a6b-4761-bca0-56314a4e376e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz namespace: kserve-ci-e2e-test resourceVersion: "14958" uid: 0b36bd82-8eeb-454e-9ef7-6d85bc801e0a kind: Event lastTimestamp: "2026-05-20T15:42:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:42:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:55Z" name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz.18b1507149dcf4ff namespace: kserve-ci-e2e-test resourceVersion: "15072" uid: 2f038577-93ad-48ad-9fba-327debd71bdf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz namespace: kserve-ci-e2e-test resourceVersion: "14958" uid: 0b36bd82-8eeb-454e-9ef7-6d85bc801e0a kind: Event lastTimestamp: "2026-05-20T15:42:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:42:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:55Z" name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz.18b150714a632ecc namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: e06fad9b-ec55-430f-bdce-5389c76f08d0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz namespace: kserve-ci-e2e-test resourceVersion: "14958" uid: 0b36bd82-8eeb-454e-9ef7-6d85bc801e0a kind: Event lastTimestamp: "2026-05-20T15:42:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T15:42:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:55Z" name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz.18b150714a702c70 namespace: kserve-ci-e2e-test resourceVersion: "15077" uid: 384c399c-7c6b-4062-97a5-5183e1aefe40 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz namespace: kserve-ci-e2e-test resourceVersion: "14958" uid: 0b36bd82-8eeb-454e-9ef7-6d85bc801e0a kind: Event lastTimestamp: "2026-05-20T15:42:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:42:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:55Z" name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz.18b150714dd1177f namespace: kserve-ci-e2e-test resourceVersion: "15079" uid: cbbcf3de-1e37-4213-a2fd-ad9d5b4a1369 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz namespace: kserve-ci-e2e-test resourceVersion: "14958" uid: 0b36bd82-8eeb-454e-9ef7-6d85bc801e0a kind: Event lastTimestamp: "2026-05-20T15:42:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:42:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:55Z" name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz.18b150714e46dbc2 namespace: kserve-ci-e2e-test resourceVersion: "15082" uid: ecb0c1fb-8866-4fc4-8474-d5024bddb9d3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-20T15:42:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz namespace: kserve-ci-e2e-test resourceVersion: "14958" uid: 0b36bd82-8eeb-454e-9ef7-6d85bc801e0a kind: Event lastTimestamp: "2026-05-20T15:44:33Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:42:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:33Z" name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz.18b15071bd9a62bd namespace: kserve-ci-e2e-test resourceVersion: "15967" uid: b29d5b2f-e053-4d9e-a92d-6edc348b94ef reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz namespace: kserve-ci-e2e-test resourceVersion: "14958" uid: 0b36bd82-8eeb-454e-9ef7-6d85bc801e0a kind: Event lastTimestamp: "2026-05-20T15:44:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T15:44:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:30Z" name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz.18b150877c30cf82 namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: 2e3a319b-3b35-47f4-933c-8a659cf03a94 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz namespace: kserve-ci-e2e-test resourceVersion: "14958" uid: 0b36bd82-8eeb-454e-9ef7-6d85bc801e0a kind: Event lastTimestamp: "2026-05-20T15:44:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:44:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:30Z" name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz.18b150877c314fd4 namespace: kserve-ci-e2e-test resourceVersion: "15928" uid: 6a227157-827c-4151-a15c-4da89d7aea05 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz namespace: kserve-ci-e2e-test resourceVersion: "14958" uid: 0b36bd82-8eeb-454e-9ef7-6d85bc801e0a kind: Event lastTimestamp: "2026-05-20T15:44:33Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp 10.134.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:33Z" name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz.18b1508817f820c3 namespace: kserve-ci-e2e-test resourceVersion: "15966" uid: 70b4b9e9-9c1c-4664-9b26-c7a50ff1b9c2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c namespace: kserve-ci-e2e-test resourceVersion: "14943" uid: 47d920a7-201a-4b93-8909-97b87ca079e2 kind: Event lastTimestamp: "2026-05-20T15:42:50Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c-ss9bz' metadata: creationTimestamp: "2026-05-20T15:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:42:50Z" name: isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c.18b150702374f928 namespace: kserve-ci-e2e-test resourceVersion: "14957" uid: d6cbec50-2a44-46ef-a7cc-656242b7a82d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-86fe3-predictor namespace: kserve-ci-e2e-test resourceVersion: "14936" uid: 5b515177-5ba7-4a88-85ea-7493934d9d79 kind: Event lastTimestamp: "2026-05-20T15:42:50Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-86fe3-predictor-786bd7b77c from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:42:50Z" name: isvc-xgboost-graph-raw-hpa-86fe3-predictor.18b150701f481ded namespace: kserve-ci-e2e-test resourceVersion: "14946" uid: 0be4008f-906b-4b17-aa1a-fe02e8d8cf59 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:43:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-86fe3-predictor namespace: kserve-ci-e2e-test resourceVersion: "14938" uid: dd4ad454-0ef2-4475-b76b-d7fcff4fff59 kind: Event lastTimestamp: "2026-05-20T15:43:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:43:20Z" name: isvc-xgboost-graph-raw-hpa-86fe3-predictor.18b150739dabd6b9 namespace: kserve-ci-e2e-test resourceVersion: "15284" uid: 74a089c3-4e51-45a1-9c0f-03ac77d408e7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:43:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-86fe3-predictor namespace: kserve-ci-e2e-test resourceVersion: "14938" uid: dd4ad454-0ef2-4475-b76b-d7fcff4fff59 kind: Event lastTimestamp: "2026-05-20T15:43:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:43:20Z" name: isvc-xgboost-graph-raw-hpa-86fe3-predictor.18b150739e1db5d4 namespace: kserve-ci-e2e-test resourceVersion: "15285" uid: bdf18dc5-435a-40c9-8e25-4dd6dc5f55f3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:43:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-86fe3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15169" uid: dd4ad454-0ef2-4475-b76b-d7fcff4fff59 kind: Event lastTimestamp: "2026-05-20T15:44:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:43:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:20Z" name: isvc-xgboost-graph-raw-hpa-86fe3-predictor.18b1507a9c8b4016 namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: 0623178b-3a82-4227-808f-e5004f8e51b0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:43:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-86fe3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15169" uid: dd4ad454-0ef2-4475-b76b-d7fcff4fff59 kind: Event lastTimestamp: "2026-05-20T15:44:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:43:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:20Z" name: isvc-xgboost-graph-raw-hpa-86fe3-predictor.18b1507a9d5ae9b6 namespace: kserve-ci-e2e-test resourceVersion: "15763" uid: 6a42ce2d-a956-427d-8456-305ac6df72d1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-20T15:42:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-86fe3 namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 43280f05-7408-46d3-ac42-b8e89e75f130 kind: Event lastTimestamp: "2026-05-20T15:44:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:44:03Z" name: isvc-xgboost-graph-raw-hpa-86fe3.18b150701ef68045 namespace: kserve-ci-e2e-test resourceVersion: "15609" uid: 0f95f0b1-f7b8-4945-bc99-35ae3696dd6c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-86fe3 namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 43280f05-7408-46d3-ac42-b8e89e75f130 kind: Event lastTimestamp: "2026-05-20T15:42:50Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-hpa-86fe3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-hpa-86fe3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:42:50Z" name: isvc-xgboost-graph-raw-hpa-86fe3.18b150701f70cfe4 namespace: kserve-ci-e2e-test resourceVersion: "14948" uid: 9371f6b8-b8f6-4edc-aa5a-1f168c2362ad reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-86fe3 namespace: kserve-ci-e2e-test resourceVersion: "15606" uid: 43280f05-7408-46d3-ac42-b8e89e75f130 kind: Event lastTimestamp: "2026-05-20T15:44:03Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-86fe3] is Ready metadata: creationTimestamp: "2026-05-20T15:44:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:44:03Z" name: isvc-xgboost-graph-raw-hpa-86fe3.18b1508123549a51 namespace: kserve-ci-e2e-test resourceVersion: "15607" uid: 68f75c89-0620-494f-ab98-d8c68cef1dda reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:44:30.544535Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-47eff-predictor-98bc6cb9d-57wjc namespace: kserve-ci-e2e-test resourceVersion: "15900" uid: bc63f01a-f82a-4ef8-b174-83adb94e1fad kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-47eff-predictor-98bc6cb9d-57wjc to ip-10-0-140-154.ec2.internal metadata: creationTimestamp: "2026-05-20T15:44:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:44:30Z" name: message-dumper-raw-47eff-predictor-98bc6cb9d-57wjc.18b15087778c4046 namespace: kserve-ci-e2e-test resourceVersion: "15905" uid: 11fef860-86f7-42a6-afab-6763e98c94e4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df74f7774-d6kd2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:30Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-47eff-predictor-98bc6cb9d-57wjc namespace: kserve-ci-e2e-test resourceVersion: "15917" uid: bc63f01a-f82a-4ef8-b174-83adb94e1fad kind: Event lastTimestamp: "2026-05-20T15:44:30Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:44:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:44:30Z" name: message-dumper-raw-47eff-predictor-98bc6cb9d-57wjc.18b1508791c26e63 namespace: kserve-ci-e2e-test resourceVersion: "15935" uid: 717f42f1-d63e-4206-a8b3-ed2568cda9de reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-47eff-predictor-98bc6cb9d-57wjc namespace: kserve-ci-e2e-test resourceVersion: "15903" uid: bc63f01a-f82a-4ef8-b174-83adb94e1fad kind: Event lastTimestamp: "2026-05-20T15:44:31Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-05-20T15:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:31Z" name: message-dumper-raw-47eff-predictor-98bc6cb9d-57wjc.18b15087931fdcee namespace: kserve-ci-e2e-test resourceVersion: "15937" uid: 1855793f-f2ba-4c9f-a2a6-486329a548e2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-47eff-predictor-98bc6cb9d-57wjc namespace: kserve-ci-e2e-test resourceVersion: "15903" uid: bc63f01a-f82a-4ef8-b174-83adb94e1fad kind: Event lastTimestamp: "2026-05-20T15:44:32Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.1s (1.1s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-05-20T15:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:32Z" name: message-dumper-raw-47eff-predictor-98bc6cb9d-57wjc.18b15087d4bf4f32 namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: 2bd91414-62da-4fde-b3e6-5f43bc02b96e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-47eff-predictor-98bc6cb9d-57wjc namespace: kserve-ci-e2e-test resourceVersion: "15903" uid: bc63f01a-f82a-4ef8-b174-83adb94e1fad kind: Event lastTimestamp: "2026-05-20T15:44:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:32Z" name: message-dumper-raw-47eff-predictor-98bc6cb9d-57wjc.18b15087d8999c17 namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: a24e751c-b3bb-4686-8b2f-6e8a7fb5d360 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-47eff-predictor-98bc6cb9d-57wjc namespace: kserve-ci-e2e-test resourceVersion: "15903" uid: bc63f01a-f82a-4ef8-b174-83adb94e1fad kind: Event lastTimestamp: "2026-05-20T15:44:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:32Z" name: message-dumper-raw-47eff-predictor-98bc6cb9d-57wjc.18b15087d915be77 namespace: kserve-ci-e2e-test resourceVersion: "15950" uid: b7a83083-c689-436a-8c7f-5bcbbef746d4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-47eff-predictor-98bc6cb9d-57wjc namespace: kserve-ci-e2e-test resourceVersion: "15903" uid: bc63f01a-f82a-4ef8-b174-83adb94e1fad kind: Event lastTimestamp: "2026-05-20T15:44:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T15:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:32Z" name: message-dumper-raw-47eff-predictor-98bc6cb9d-57wjc.18b15087d92498ce namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: daaa4267-b811-47cf-b2c8-7f8c93d42434 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-47eff-predictor-98bc6cb9d-57wjc namespace: kserve-ci-e2e-test resourceVersion: "15903" uid: bc63f01a-f82a-4ef8-b174-83adb94e1fad kind: Event lastTimestamp: "2026-05-20T15:44:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:32Z" name: message-dumper-raw-47eff-predictor-98bc6cb9d-57wjc.18b15087dd3ca3fe namespace: kserve-ci-e2e-test resourceVersion: "15954" uid: 887f2277-5943-4ea5-83c9-b3ae007002cc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-47eff-predictor-98bc6cb9d-57wjc namespace: kserve-ci-e2e-test resourceVersion: "15903" uid: bc63f01a-f82a-4ef8-b174-83adb94e1fad kind: Event lastTimestamp: "2026-05-20T15:44:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:32Z" name: message-dumper-raw-47eff-predictor-98bc6cb9d-57wjc.18b15087ddb778a7 namespace: kserve-ci-e2e-test resourceVersion: "15955" uid: 075cfd2e-285b-4424-a6ed-7a42d129ccf3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-47eff-predictor-98bc6cb9d-57wjc namespace: kserve-ci-e2e-test resourceVersion: "15903" uid: bc63f01a-f82a-4ef8-b174-83adb94e1fad kind: Event lastTimestamp: "2026-05-20T15:46:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T15:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:46:05Z" name: message-dumper-raw-47eff-predictor-98bc6cb9d-57wjc.18b1509da957391a namespace: kserve-ci-e2e-test resourceVersion: "16806" uid: e3bb9a1f-2ef6-4c13-95ad-b812c7d43519 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-47eff-predictor-98bc6cb9d-57wjc namespace: kserve-ci-e2e-test resourceVersion: "15903" uid: bc63f01a-f82a-4ef8-b174-83adb94e1fad kind: Event lastTimestamp: "2026-05-20T15:46:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:46:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:46:05Z" name: message-dumper-raw-47eff-predictor-98bc6cb9d-57wjc.18b1509da9578d13 namespace: kserve-ci-e2e-test resourceVersion: "16807" uid: e3d7c333-501a-47e8-a372-f1dd3a0f0c67 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-47eff-predictor-98bc6cb9d namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: e84914bd-0e55-4c8e-a5f1-157344edf8e4 kind: Event lastTimestamp: "2026-05-20T15:44:30Z" message: 'Created pod: message-dumper-raw-47eff-predictor-98bc6cb9d-57wjc' metadata: creationTimestamp: "2026-05-20T15:44:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:30Z" name: message-dumper-raw-47eff-predictor-98bc6cb9d.18b1508776f5301a namespace: kserve-ci-e2e-test resourceVersion: "15902" uid: 89c899b0-8221-4298-bda7-5d25f9468f1a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-47eff-predictor namespace: kserve-ci-e2e-test resourceVersion: "15872" uid: 6ae72442-a4fd-4675-b7ad-8c1c4a0aad64 kind: Event lastTimestamp: "2026-05-20T15:44:30Z" message: Scaled up replica set message-dumper-raw-47eff-predictor-98bc6cb9d from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:44:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:30Z" name: message-dumper-raw-47eff-predictor.18b150877285b47a namespace: kserve-ci-e2e-test resourceVersion: "15885" uid: fea753ab-ca2f-4a55-88ad-e789035caad8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-47eff-predictor namespace: kserve-ci-e2e-test resourceVersion: "15879" uid: fac64a3c-96a1-4238-9988-1b70d10f5697 kind: Event lastTimestamp: "2026-05-20T15:44:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:44:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:45Z" name: message-dumper-raw-47eff-predictor.18b1508af1d2411b namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: cc662bff-72ae-4a85-9f78-e73cd4fdf595 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-47eff-predictor namespace: kserve-ci-e2e-test resourceVersion: "15879" uid: fac64a3c-96a1-4238-9988-1b70d10f5697 kind: Event lastTimestamp: "2026-05-20T15:44:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:44:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:45Z" name: message-dumper-raw-47eff-predictor.18b1508af24c62d8 namespace: kserve-ci-e2e-test resourceVersion: "16126" uid: 3be769dd-b1dc-4d57-8816-f647f920cf9f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-47eff-predictor namespace: kserve-ci-e2e-test resourceVersion: "16125" uid: fac64a3c-96a1-4238-9988-1b70d10f5697 kind: Event lastTimestamp: "2026-05-20T15:45:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:45:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:00Z" name: message-dumper-raw-47eff-predictor.18b1508e7114f8cc namespace: kserve-ci-e2e-test resourceVersion: "16258" uid: 49fabb91-4480-4421-992c-b69f23790a86 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-47eff-predictor namespace: kserve-ci-e2e-test resourceVersion: "16125" uid: fac64a3c-96a1-4238-9988-1b70d10f5697 kind: Event lastTimestamp: "2026-05-20T15:45:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:45:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:00Z" name: message-dumper-raw-47eff-predictor.18b1508e7186bc03 namespace: kserve-ci-e2e-test resourceVersion: "16260" uid: 322a13cd-bbd8-4541-b942-f3b6fb9a1192 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-20T15:44:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-47eff namespace: kserve-ci-e2e-test resourceVersion: "15861" uid: dc60765a-d449-4469-a546-fa95e078f4e2 kind: Event lastTimestamp: "2026-05-20T15:44:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:44:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:44:39Z" name: message-dumper-raw-47eff.18b150877262e0a6 namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 4e9528a4-7d64-47e6-8db6-c1db287c2549 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-47eff namespace: kserve-ci-e2e-test resourceVersion: "15861" uid: dc60765a-d449-4469-a546-fa95e078f4e2 kind: Event lastTimestamp: "2026-05-20T15:44:30Z" message: 'Failed to update status for InferenceService "message-dumper-raw-47eff": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-47eff": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:44:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:44:30Z" name: message-dumper-raw-47eff.18b1508772fadc53 namespace: kserve-ci-e2e-test resourceVersion: "15889" uid: f2defb58-34c6-4aae-b159-5ce154482dc8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-47eff namespace: kserve-ci-e2e-test resourceVersion: "16032" uid: dc60765a-d449-4469-a546-fa95e078f4e2 kind: Event lastTimestamp: "2026-05-20T15:44:39Z" message: InferenceService [message-dumper-raw-47eff] is Ready metadata: creationTimestamp: "2026-05-20T15:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:44:39Z" name: message-dumper-raw-47eff.18b1508997a2e967 namespace: kserve-ci-e2e-test resourceVersion: "16033" uid: 99441041-1c7f-4dea-befa-bfcfe057a7c5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:42:39.853132Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-61d94-f98c8d7cb-5qwz7 namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: 0352136f-4b78-41c2-aa76-212313a1dfc7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-61d94-f98c8d7cb-5qwz7 to ip-10-0-137-63.ec2.internal metadata: creationTimestamp: "2026-05-20T15:42:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:42:39Z" name: model-chainer-raw-61d94-f98c8d7cb-5qwz7.18b1506db1d376d8 namespace: kserve-ci-e2e-test resourceVersion: "14767" uid: e4d8e263-7e0d-493c-a5ea-f6284caf01f6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df74f7774-d6kd2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:40Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-61d94-f98c8d7cb-5qwz7 namespace: kserve-ci-e2e-test resourceVersion: "14765" uid: 0352136f-4b78-41c2-aa76-212313a1dfc7 kind: Event lastTimestamp: "2026-05-20T15:42:40Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-61d94-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T15:42:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:40Z" name: model-chainer-raw-61d94-f98c8d7cb-5qwz7.18b1506dc0b483ab namespace: kserve-ci-e2e-test resourceVersion: "14777" uid: a59d40ec-6481-4bbc-b28d-4076fd747b4f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-63.ec2.internal source: component: kubelet host: ip-10-0-137-63.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:40Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-61d94-f98c8d7cb-5qwz7 namespace: kserve-ci-e2e-test resourceVersion: "14771" uid: 0352136f-4b78-41c2-aa76-212313a1dfc7 kind: Event lastTimestamp: "2026-05-20T15:42:40Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:42:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:42:40Z" name: model-chainer-raw-61d94-f98c8d7cb-5qwz7.18b1506def0d02cd namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 8060094d-1bcb-4bca-b422-e2cc80e0946a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-61d94} kind: Pod name: model-chainer-raw-61d94-f98c8d7cb-5qwz7 namespace: kserve-ci-e2e-test resourceVersion: "14765" uid: 0352136f-4b78-41c2-aa76-212313a1dfc7 kind: Event lastTimestamp: "2026-05-20T15:42:40Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:88955c235550cb8e8a9cbe850692d2683b6dea68a08743540305f46e732a59a3" metadata: creationTimestamp: "2026-05-20T15:42:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:40Z" name: model-chainer-raw-61d94-f98c8d7cb-5qwz7.18b1506df0b69294 namespace: kserve-ci-e2e-test resourceVersion: "14788" uid: c772abad-2d67-4b97-9d64-5beae80d0ccc reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-63.ec2.internal source: component: kubelet host: ip-10-0-137-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-61d94} kind: Pod name: model-chainer-raw-61d94-f98c8d7cb-5qwz7 namespace: kserve-ci-e2e-test resourceVersion: "14765" uid: 0352136f-4b78-41c2-aa76-212313a1dfc7 kind: Event lastTimestamp: "2026-05-20T15:42:43Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:88955c235550cb8e8a9cbe850692d2683b6dea68a08743540305f46e732a59a3" in 2.277s (2.277s including waiting). Image size: 218002168 bytes.' metadata: creationTimestamp: "2026-05-20T15:42:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:43Z" name: model-chainer-raw-61d94-f98c8d7cb-5qwz7.18b1506e787346e2 namespace: kserve-ci-e2e-test resourceVersion: "14805" uid: 0195d4b3-0c97-4013-89ee-56e3d626c0ba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-63.ec2.internal source: component: kubelet host: ip-10-0-137-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-61d94} kind: Pod name: model-chainer-raw-61d94-f98c8d7cb-5qwz7 namespace: kserve-ci-e2e-test resourceVersion: "14765" uid: 0352136f-4b78-41c2-aa76-212313a1dfc7 kind: Event lastTimestamp: "2026-05-20T15:42:43Z" message: 'Created container: model-chainer-raw-61d94' metadata: creationTimestamp: "2026-05-20T15:42:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:43Z" name: model-chainer-raw-61d94-f98c8d7cb-5qwz7.18b1506e7b29dfb8 namespace: kserve-ci-e2e-test resourceVersion: "14806" uid: 6093cd59-7cf7-45b0-837a-8d53ef6ee943 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-63.ec2.internal source: component: kubelet host: ip-10-0-137-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-61d94} kind: Pod name: model-chainer-raw-61d94-f98c8d7cb-5qwz7 namespace: kserve-ci-e2e-test resourceVersion: "14765" uid: 0352136f-4b78-41c2-aa76-212313a1dfc7 kind: Event lastTimestamp: "2026-05-20T15:42:43Z" message: Started container model-chainer-raw-61d94 metadata: creationTimestamp: "2026-05-20T15:42:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:43Z" name: model-chainer-raw-61d94-f98c8d7cb-5qwz7.18b1506e7b909e48 namespace: kserve-ci-e2e-test resourceVersion: "14807" uid: b2e71c80-a37d-4a4a-92dd-54a1edf18f84 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-63.ec2.internal source: component: kubelet host: ip-10-0-137-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-61d94} kind: Pod name: model-chainer-raw-61d94-f98c8d7cb-5qwz7 namespace: kserve-ci-e2e-test resourceVersion: "14765" uid: 0352136f-4b78-41c2-aa76-212313a1dfc7 kind: Event lastTimestamp: "2026-05-20T15:42:49Z" message: Stopping container model-chainer-raw-61d94 metadata: creationTimestamp: "2026-05-20T15:42:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:42:49Z" name: model-chainer-raw-61d94-f98c8d7cb-5qwz7.18b150700ddbc9c6 namespace: kserve-ci-e2e-test resourceVersion: "14880" uid: 569c6c44-6344-422b-b4b5-1e7eacdc8aa2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-63.ec2.internal source: component: kubelet host: ip-10-0-137-63.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T15:42:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-61d94} kind: Pod name: model-chainer-raw-61d94-f98c8d7cb-5qwz7 namespace: kserve-ci-e2e-test resourceVersion: "14765" uid: 0352136f-4b78-41c2-aa76-212313a1dfc7 kind: Event lastTimestamp: "2026-05-20T15:43:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T15:42:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:19Z" name: model-chainer-raw-61d94-f98c8d7cb-5qwz7.18b1507117c8fe0b namespace: kserve-ci-e2e-test resourceVersion: "15274" uid: fe6a5241-40ee-430e-b047-8a1f6b25dd08 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-63.ec2.internal source: component: kubelet host: ip-10-0-137-63.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-61d94-f98c8d7cb namespace: kserve-ci-e2e-test resourceVersion: "14759" uid: f511ba70-1747-4edc-8d28-8b43cb111d5e kind: Event lastTimestamp: "2026-05-20T15:42:39Z" message: 'Created pod: model-chainer-raw-61d94-f98c8d7cb-5qwz7' metadata: creationTimestamp: "2026-05-20T15:42:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:42:39Z" name: model-chainer-raw-61d94-f98c8d7cb.18b1506db135afd5 namespace: kserve-ci-e2e-test resourceVersion: "14764" uid: c7d25913-414e-4500-a025-da6207e7b8f1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-61d94 namespace: kserve-ci-e2e-test resourceVersion: "14757" uid: 43195cd3-eac1-438a-bbeb-f5b1a4fca2d2 kind: Event lastTimestamp: "2026-05-20T15:42:39Z" message: Scaled up replica set model-chainer-raw-61d94-f98c8d7cb from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:42:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:42:39Z" name: model-chainer-raw-61d94.18b1506db01c6156 namespace: kserve-ci-e2e-test resourceVersion: "14760" uid: 90c10af8-25d1-42af-9a39-8580df1e8fca reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:42:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-61d94 namespace: kserve-ci-e2e-test resourceVersion: "14773" uid: 093f0ca9-831f-4e3a-834d-2b9a3bc2cc77 kind: Event lastTimestamp: "2026-05-20T15:42:39Z" message: InferenceGraph [model-chainer-raw-61d94] is Ready metadata: creationTimestamp: "2026-05-20T15:42:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:42:39Z" name: model-chainer-raw-61d94.18b1506db6d70645 namespace: kserve-ci-e2e-test resourceVersion: "14775" uid: f3d3f82a-05e7-4914-b965-598cf2055844 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:44:20.263648Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-86fe3-8685d77bfb-ml8gs namespace: kserve-ci-e2e-test resourceVersion: "15743" uid: 46839c31-3415-4fea-b595-e734ca411d93 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-86fe3-8685d77bfb-ml8gs to ip-10-0-137-63.ec2.internal metadata: creationTimestamp: "2026-05-20T15:44:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:44:20Z" name: model-chainer-raw-hpa-86fe3-8685d77bfb-ml8gs.18b1508512c26070 namespace: kserve-ci-e2e-test resourceVersion: "15748" uid: baf69885-2112-4307-892b-658c83bd6cf0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df74f7774-d6kd2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:20Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-86fe3-8685d77bfb-ml8gs namespace: kserve-ci-e2e-test resourceVersion: "15752" uid: 46839c31-3415-4fea-b595-e734ca411d93 kind: Event lastTimestamp: "2026-05-20T15:44:20Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:44:20Z" name: model-chainer-raw-hpa-86fe3-8685d77bfb-ml8gs.18b150852ca0a62e namespace: kserve-ci-e2e-test resourceVersion: "15765" uid: a9ddbb6b-d049-4909-8b47-c0d7003dc6aa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-86fe3} kind: Pod name: model-chainer-raw-hpa-86fe3-8685d77bfb-ml8gs namespace: kserve-ci-e2e-test resourceVersion: "15746" uid: 46839c31-3415-4fea-b595-e734ca411d93 kind: Event lastTimestamp: "2026-05-20T15:44:20Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:88955c235550cb8e8a9cbe850692d2683b6dea68a08743540305f46e732a59a3" already present on machine metadata: creationTimestamp: "2026-05-20T15:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:20Z" name: model-chainer-raw-hpa-86fe3-8685d77bfb-ml8gs.18b150852de4d36e namespace: kserve-ci-e2e-test resourceVersion: "15768" uid: a1e7b305-6071-44bc-8570-1a056bce4c3a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-63.ec2.internal source: component: kubelet host: ip-10-0-137-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-86fe3} kind: Pod name: model-chainer-raw-hpa-86fe3-8685d77bfb-ml8gs namespace: kserve-ci-e2e-test resourceVersion: "15746" uid: 46839c31-3415-4fea-b595-e734ca411d93 kind: Event lastTimestamp: "2026-05-20T15:44:20Z" message: 'Created container: model-chainer-raw-hpa-86fe3' metadata: creationTimestamp: "2026-05-20T15:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:20Z" name: model-chainer-raw-hpa-86fe3-8685d77bfb-ml8gs.18b15085312ccf42 namespace: kserve-ci-e2e-test resourceVersion: "15769" uid: c46daaf7-ed0b-46af-93b4-98d5034c10f4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-63.ec2.internal source: component: kubelet host: ip-10-0-137-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-86fe3} kind: Pod name: model-chainer-raw-hpa-86fe3-8685d77bfb-ml8gs namespace: kserve-ci-e2e-test resourceVersion: "15746" uid: 46839c31-3415-4fea-b595-e734ca411d93 kind: Event lastTimestamp: "2026-05-20T15:44:20Z" message: Started container model-chainer-raw-hpa-86fe3 metadata: creationTimestamp: "2026-05-20T15:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:20Z" name: model-chainer-raw-hpa-86fe3-8685d77bfb-ml8gs.18b1508531a7848b namespace: kserve-ci-e2e-test resourceVersion: "15770" uid: fb15f973-f5bf-4429-bb3f-7d787646ae3f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-63.ec2.internal source: component: kubelet host: ip-10-0-137-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-86fe3} kind: Pod name: model-chainer-raw-hpa-86fe3-8685d77bfb-ml8gs namespace: kserve-ci-e2e-test resourceVersion: "15746" uid: 46839c31-3415-4fea-b595-e734ca411d93 kind: Event lastTimestamp: "2026-05-20T15:44:30Z" message: Stopping container model-chainer-raw-hpa-86fe3 metadata: creationTimestamp: "2026-05-20T15:44:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:30Z" name: model-chainer-raw-hpa-86fe3-8685d77bfb-ml8gs.18b150876a85c947 namespace: kserve-ci-e2e-test resourceVersion: "15857" uid: 40eb0b03-85f9-4210-90ac-8a4c20ebedba reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-63.ec2.internal source: component: kubelet host: ip-10-0-137-63.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T15:44:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-86fe3} kind: Pod name: model-chainer-raw-hpa-86fe3-8685d77bfb-ml8gs namespace: kserve-ci-e2e-test resourceVersion: "15746" uid: 46839c31-3415-4fea-b595-e734ca411d93 kind: Event lastTimestamp: "2026-05-20T15:44:57Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T15:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:57Z" name: model-chainer-raw-hpa-86fe3-8685d77bfb-ml8gs.18b15087fbd00f02 namespace: kserve-ci-e2e-test resourceVersion: "16235" uid: a45dc9f9-cd2b-49f4-9f9c-82458b24ad32 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-63.ec2.internal source: component: kubelet host: ip-10-0-137-63.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-86fe3-8685d77bfb namespace: kserve-ci-e2e-test resourceVersion: "15740" uid: 282a435a-771f-4071-9fe9-87923e605f8d kind: Event lastTimestamp: "2026-05-20T15:44:20Z" message: 'Created pod: model-chainer-raw-hpa-86fe3-8685d77bfb-ml8gs' metadata: creationTimestamp: "2026-05-20T15:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:20Z" name: model-chainer-raw-hpa-86fe3-8685d77bfb.18b150851200dc3d namespace: kserve-ci-e2e-test resourceVersion: "15745" uid: d9c82a1b-bdf7-4797-b070-5e13286de000 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-86fe3 namespace: kserve-ci-e2e-test resourceVersion: "15738" uid: b3a93333-af70-4f1a-a370-44d44328703c kind: Event lastTimestamp: "2026-05-20T15:44:20Z" message: Scaled up replica set model-chainer-raw-hpa-86fe3-8685d77bfb from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:20Z" name: model-chainer-raw-hpa-86fe3.18b1508511396381 namespace: kserve-ci-e2e-test resourceVersion: "15741" uid: bea15ada-0a23-4105-80ce-174304eb0b81 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-86fe3 namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: 57ba96c4-7503-467b-8326-8dedcdda48cb kind: Event lastTimestamp: "2026-05-20T15:44:20Z" message: InferenceGraph [model-chainer-raw-hpa-86fe3] is Ready metadata: creationTimestamp: "2026-05-20T15:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:44:20Z" name: model-chainer-raw-hpa-86fe3.18b1508516f03e55 namespace: kserve-ci-e2e-test resourceVersion: "15760" uid: abb2cf4f-7f51-45e2-85ac-917c784e2a22 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:50:25.121094Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4 namespace: kserve-ci-e2e-test resourceVersion: "19003" uid: e10254b7-f583-44f7-a7e5-a2184551333d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4 to ip-10-0-140-154.ec2.internal metadata: creationTimestamp: "2026-05-20T15:50:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:50:25Z" name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4.18b150da05f54dfa namespace: kserve-ci-e2e-test resourceVersion: "19007" uid: 1c80bb3e-cfe2-44c5-aca1-d4047389a36b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df74f7774-d6kd2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:50:25Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4 namespace: kserve-ci-e2e-test resourceVersion: "19011" uid: e10254b7-f583-44f7-a7e5-a2184551333d kind: Event lastTimestamp: "2026-05-20T15:50:25Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:50:25Z" name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4.18b150da1fd7bae6 namespace: kserve-ci-e2e-test resourceVersion: "19015" uid: 4f08ad73-6e90-4d29-92a7-bd644df3736a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:50:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4 namespace: kserve-ci-e2e-test resourceVersion: "19005" uid: e10254b7-f583-44f7-a7e5-a2184551333d kind: Event lastTimestamp: "2026-05-20T15:50:25Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:050b18baea3a969ddc2a15671f898a5f40ff53eb89f54c18856b130dc10bd894" already present on machine metadata: creationTimestamp: "2026-05-20T15:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:50:25Z" name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4.18b150da2186e4d6 namespace: kserve-ci-e2e-test resourceVersion: "19017" uid: b03415d0-6f4d-47d7-a93e-3c9e3233cdcc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:50:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4 namespace: kserve-ci-e2e-test resourceVersion: "19005" uid: e10254b7-f583-44f7-a7e5-a2184551333d kind: Event lastTimestamp: "2026-05-20T15:50:25Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T15:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:50:25Z" name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4.18b150da24f25fc4 namespace: kserve-ci-e2e-test resourceVersion: "19018" uid: 643cc0ed-b452-4bba-894e-4f05b7727b48 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:50:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4 namespace: kserve-ci-e2e-test resourceVersion: "19005" uid: e10254b7-f583-44f7-a7e5-a2184551333d kind: Event lastTimestamp: "2026-05-20T15:50:25Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T15:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:50:25Z" name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4.18b150da256f2e57 namespace: kserve-ci-e2e-test resourceVersion: "19019" uid: 44582b3e-aed6-43a2-8649-dbf84b45d32b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:50:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4 namespace: kserve-ci-e2e-test resourceVersion: "19005" uid: e10254b7-f583-44f7-a7e5-a2184551333d kind: Event lastTimestamp: "2026-05-20T15:50:29Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-05-20T15:50:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:50:29Z" name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4.18b150db2026d8dc namespace: kserve-ci-e2e-test resourceVersion: "19060" uid: 960a7365-d158-4f34-aa49-07e401161ca2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:50:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4 namespace: kserve-ci-e2e-test resourceVersion: "19005" uid: e10254b7-f583-44f7-a7e5-a2184551333d kind: Event lastTimestamp: "2026-05-20T15:50:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:50:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:50:29Z" name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4.18b150db236d5ae0 namespace: kserve-ci-e2e-test resourceVersion: "19062" uid: 56b232f7-d8ca-45ab-a196-9ae1b7fa2709 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:50:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4 namespace: kserve-ci-e2e-test resourceVersion: "19005" uid: e10254b7-f583-44f7-a7e5-a2184551333d kind: Event lastTimestamp: "2026-05-20T15:50:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:50:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:50:29Z" name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4.18b150db23e12041 namespace: kserve-ci-e2e-test resourceVersion: "19065" uid: 37db6b1b-3b75-4c72-a813-d743cddbfcd3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:50:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4 namespace: kserve-ci-e2e-test resourceVersion: "19005" uid: e10254b7-f583-44f7-a7e5-a2184551333d kind: Event lastTimestamp: "2026-05-20T15:50:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T15:50:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:50:29Z" name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4.18b150db23ed3990 namespace: kserve-ci-e2e-test resourceVersion: "19066" uid: 6f81758d-aead-44ed-a882-a94a9ac2f623 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:50:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4 namespace: kserve-ci-e2e-test resourceVersion: "19005" uid: e10254b7-f583-44f7-a7e5-a2184551333d kind: Event lastTimestamp: "2026-05-20T15:50:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:50:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:50:29Z" name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4.18b150db272e9043 namespace: kserve-ci-e2e-test resourceVersion: "19068" uid: 177a40b4-23d9-4f5e-973e-112ca90519a8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:50:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4 namespace: kserve-ci-e2e-test resourceVersion: "19005" uid: e10254b7-f583-44f7-a7e5-a2184551333d kind: Event lastTimestamp: "2026-05-20T15:50:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:50:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:50:29Z" name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4.18b150db27a7a725 namespace: kserve-ci-e2e-test resourceVersion: "19069" uid: 398d05ef-05d3-428e-8927-ad93a4f8ad7e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-20T15:50:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4 namespace: kserve-ci-e2e-test resourceVersion: "19005" uid: e10254b7-f583-44f7-a7e5-a2184551333d kind: Event lastTimestamp: "2026-05-20T15:51:46Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:50:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:51:46Z" name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4.18b150db5c1eacb7 namespace: kserve-ci-e2e-test resourceVersion: "19684" uid: ce1dcbe8-e4d2-4940-995b-4f23f787935c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:51:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4 namespace: kserve-ci-e2e-test resourceVersion: "19005" uid: e10254b7-f583-44f7-a7e5-a2184551333d kind: Event lastTimestamp: "2026-05-20T15:51:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T15:51:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:51:44Z" name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4.18b150ec8c2d417d namespace: kserve-ci-e2e-test resourceVersion: "19628" uid: 4232c995-bd21-4eba-8c0f-8953279d9fe6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:51:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4 namespace: kserve-ci-e2e-test resourceVersion: "19005" uid: e10254b7-f583-44f7-a7e5-a2184551333d kind: Event lastTimestamp: "2026-05-20T15:51:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:51:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:51:44Z" name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4.18b150ec8c2e1403 namespace: kserve-ci-e2e-test resourceVersion: "19630" uid: fee47394-37fd-49bc-bf36-36678e4b88b5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:51:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4 namespace: kserve-ci-e2e-test resourceVersion: "19005" uid: e10254b7-f583-44f7-a7e5-a2184551333d kind: Event lastTimestamp: "2026-05-20T15:51:46Z" message: 'Readiness probe failed: Get "https://10.134.0.39:8643/healthz": dial tcp 10.134.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:51:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:51:46Z" name: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4.18b150ed0e4314f5 namespace: kserve-ci-e2e-test resourceVersion: "19683" uid: 805d8446-d90b-41ac-a59f-2744932fcb3f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:50:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-312a7-predictor-666cb5fbf5 namespace: kserve-ci-e2e-test resourceVersion: "18986" uid: f4357716-86a0-4767-86f3-678d1af295bc kind: Event lastTimestamp: "2026-05-20T15:50:25Z" message: 'Created pod: raw-sklearn-312a7-predictor-666cb5fbf5-g4qg4' metadata: creationTimestamp: "2026-05-20T15:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:50:25Z" name: raw-sklearn-312a7-predictor-666cb5fbf5.18b150da0588f63a namespace: kserve-ci-e2e-test resourceVersion: "19004" uid: 0772717a-73ee-4958-a559-999b25cd461b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:50:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-312a7-predictor namespace: kserve-ci-e2e-test resourceVersion: "18984" uid: 047440b3-1cf5-4771-9053-d58d0015c97b kind: Event lastTimestamp: "2026-05-20T15:50:24Z" message: Scaled up replica set raw-sklearn-312a7-predictor-666cb5fbf5 from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:50:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:50:24Z" name: raw-sklearn-312a7-predictor.18b150d9ec5a5fc3 namespace: kserve-ci-e2e-test resourceVersion: "18988" uid: 650f40f4-58d0-430f-8d03-ebc4797efd6a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:50:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-312a7-predictor namespace: kserve-ci-e2e-test resourceVersion: "18985" uid: c0e5c5a5-09d5-42a8-b3b0-1bf50bc68ead kind: Event lastTimestamp: "2026-05-20T15:50:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:50:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:50:54Z" name: raw-sklearn-312a7-predictor.18b150dd6b51696a namespace: kserve-ci-e2e-test resourceVersion: "19244" uid: fae2e646-aba4-4633-8eaa-1c8b2db919dc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:50:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-312a7-predictor namespace: kserve-ci-e2e-test resourceVersion: "18985" uid: c0e5c5a5-09d5-42a8-b3b0-1bf50bc68ead kind: Event lastTimestamp: "2026-05-20T15:50:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:50:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:50:54Z" name: raw-sklearn-312a7-predictor.18b150dd6bae0c94 namespace: kserve-ci-e2e-test resourceVersion: "19245" uid: eedf7582-ea17-4e75-8c92-1400bc311974 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:51:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-312a7-predictor namespace: kserve-ci-e2e-test resourceVersion: "19143" uid: c0e5c5a5-09d5-42a8-b3b0-1bf50bc68ead kind: Event lastTimestamp: "2026-05-20T15:51:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:51:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:51:39Z" name: raw-sklearn-312a7-predictor.18b150e469c749bc namespace: kserve-ci-e2e-test resourceVersion: "19570" uid: 4477ab21-ac3b-4cae-a6dc-71fd6cd4e913 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:51:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-312a7-predictor namespace: kserve-ci-e2e-test resourceVersion: "19143" uid: c0e5c5a5-09d5-42a8-b3b0-1bf50bc68ead kind: Event lastTimestamp: "2026-05-20T15:51:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:51:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:51:39Z" name: raw-sklearn-312a7-predictor.18b150e46a2ed32d namespace: kserve-ci-e2e-test resourceVersion: "19571" uid: b0d494c8-13c4-4efc-8e3a-51b0b8838441 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-20T15:50:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-312a7 namespace: kserve-ci-e2e-test resourceVersion: "18973" uid: b7a98383-a110-41cb-bff3-cba6553638cb kind: Event lastTimestamp: "2026-05-20T15:51:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:50:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:51:37Z" name: raw-sklearn-312a7.18b150d9ec267f2a namespace: kserve-ci-e2e-test resourceVersion: "19550" uid: f3e12f76-bb3a-4458-bfb8-df498ba79c7f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:50:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-312a7 namespace: kserve-ci-e2e-test resourceVersion: "18973" uid: b7a98383-a110-41cb-bff3-cba6553638cb kind: Event lastTimestamp: "2026-05-20T15:50:24Z" message: 'Failed to update status for InferenceService "raw-sklearn-312a7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-312a7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:50:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:50:24Z" name: raw-sklearn-312a7.18b150d9ecd0793f namespace: kserve-ci-e2e-test resourceVersion: "18997" uid: 6f58edec-ffcc-4093-a149-2b074394e682 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:51:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-312a7 namespace: kserve-ci-e2e-test resourceVersion: "19546" uid: b7a98383-a110-41cb-bff3-cba6553638cb kind: Event lastTimestamp: "2026-05-20T15:51:36Z" message: InferenceService [raw-sklearn-312a7] is Ready metadata: creationTimestamp: "2026-05-20T15:51:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:51:36Z" name: raw-sklearn-312a7.18b150eac02aeeb4 namespace: kserve-ci-e2e-test resourceVersion: "19547" uid: 3b039c61-dce7-4b50-ad79-1b755da90d2e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:51:44.781496Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs namespace: kserve-ci-e2e-test resourceVersion: "19646" uid: 03b83c05-8f66-405b-a6f3-fae7d47e42b8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs to ip-10-0-140-154.ec2.internal metadata: creationTimestamp: "2026-05-20T15:51:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:51:44Z" name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs.18b150ec92168d84 namespace: kserve-ci-e2e-test resourceVersion: "19650" uid: da87bc8d-d711-4620-b5ab-d6c82ffb8263 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6df74f7774-d6kd2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:51:45Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs namespace: kserve-ci-e2e-test resourceVersion: "19654" uid: 03b83c05-8f66-405b-a6f3-fae7d47e42b8 kind: Event lastTimestamp: "2026-05-20T15:51:45Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:51:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:51:45Z" name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs.18b150ecabb61900 namespace: kserve-ci-e2e-test resourceVersion: "19663" uid: abc8fdad-202f-4d01-adbc-2618c18c02f8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:51:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: 03b83c05-8f66-405b-a6f3-fae7d47e42b8 kind: Event lastTimestamp: "2026-05-20T15:51:45Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:050b18baea3a969ddc2a15671f898a5f40ff53eb89f54c18856b130dc10bd894" already present on machine metadata: creationTimestamp: "2026-05-20T15:51:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:51:45Z" name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs.18b150ecad16a66f namespace: kserve-ci-e2e-test resourceVersion: "19665" uid: 6cc2a7b6-40c0-4806-ba0f-fee7a651d2d9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:51:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: 03b83c05-8f66-405b-a6f3-fae7d47e42b8 kind: Event lastTimestamp: "2026-05-20T15:51:45Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T15:51:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:51:45Z" name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs.18b150ecaff6b520 namespace: kserve-ci-e2e-test resourceVersion: "19666" uid: b7453a7d-2e2e-4c99-8e85-335e1881a153 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:51:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: 03b83c05-8f66-405b-a6f3-fae7d47e42b8 kind: Event lastTimestamp: "2026-05-20T15:51:45Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T15:51:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:51:45Z" name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs.18b150ecb06422a8 namespace: kserve-ci-e2e-test resourceVersion: "19667" uid: a77cb359-d70b-4dbf-a512-3815f4aeda32 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:51:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: 03b83c05-8f66-405b-a6f3-fae7d47e42b8 kind: Event lastTimestamp: "2026-05-20T15:51:49Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-05-20T15:51:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:51:49Z" name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs.18b150ed948ccf0a namespace: kserve-ci-e2e-test resourceVersion: "19704" uid: b2cb2e99-1f7b-43cb-90c6-5b7e8cca49f1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:51:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: 03b83c05-8f66-405b-a6f3-fae7d47e42b8 kind: Event lastTimestamp: "2026-05-20T15:51:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:51:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:51:49Z" name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs.18b150ed97e1081d namespace: kserve-ci-e2e-test resourceVersion: "19708" uid: 2c8575d0-1407-4aa0-9da3-08dc2ac8b8ef reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:51:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: 03b83c05-8f66-405b-a6f3-fae7d47e42b8 kind: Event lastTimestamp: "2026-05-20T15:51:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:51:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:51:49Z" name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs.18b150ed985e17a3 namespace: kserve-ci-e2e-test resourceVersion: "19711" uid: 4e2db77c-52d1-4f09-864a-97b38e392c57 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:51:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: 03b83c05-8f66-405b-a6f3-fae7d47e42b8 kind: Event lastTimestamp: "2026-05-20T15:51:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T15:51:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:51:49Z" name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs.18b150ed986ddbca namespace: kserve-ci-e2e-test resourceVersion: "19712" uid: 3e95fba2-f5ff-4b17-a755-e9ec069e8b52 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:51:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: 03b83c05-8f66-405b-a6f3-fae7d47e42b8 kind: Event lastTimestamp: "2026-05-20T15:51:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:51:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:51:49Z" name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs.18b150ed9bac0316 namespace: kserve-ci-e2e-test resourceVersion: "19715" uid: 4668acd3-010f-466b-9804-227335c0d1d5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:51:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: 03b83c05-8f66-405b-a6f3-fae7d47e42b8 kind: Event lastTimestamp: "2026-05-20T15:51:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:51:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:51:49Z" name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs.18b150ed9c2a4990 namespace: kserve-ci-e2e-test resourceVersion: "19716" uid: 9c983f7b-a92a-469b-966f-c6a50fcc29f2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-20T15:51:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: 03b83c05-8f66-405b-a6f3-fae7d47e42b8 kind: Event lastTimestamp: "2026-05-20T15:52:46Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:51:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:52:46Z" name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs.18b150edd0887faf namespace: kserve-ci-e2e-test resourceVersion: "20123" uid: 3dd4243b-96e1-4749-b281-53d83f535390 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:53:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: 03b83c05-8f66-405b-a6f3-fae7d47e42b8 kind: Event lastTimestamp: "2026-05-20T15:53:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T15:53:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:53:06Z" name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs.18b150ff84378f7c namespace: kserve-ci-e2e-test resourceVersion: "20292" uid: f288d4d7-1fb0-4745-a2ce-a9b7d91e80e7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:53:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: 03b83c05-8f66-405b-a6f3-fae7d47e42b8 kind: Event lastTimestamp: "2026-05-20T15:53:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:53:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:53:06Z" name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs.18b150ff8438daf4 namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 7abf320a-695c-40e0-ad82-59dcc39643c1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs namespace: kserve-ci-e2e-test resourceVersion: "19648" uid: 03b83c05-8f66-405b-a6f3-fae7d47e42b8 kind: Event lastTimestamp: "2026-05-20T15:53:11Z" message: 'Readiness probe failed: Get "https://10.134.0.40:8643/healthz": dial tcp 10.134.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:53:11Z" name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs.18b15100acbb1cc8 namespace: kserve-ci-e2e-test resourceVersion: "20360" uid: cf59a7c2-f599-4e96-8ca5-a6990ab85549 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-154.ec2.internal source: component: kubelet host: ip-10-0-140-154.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:51:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49 namespace: kserve-ci-e2e-test resourceVersion: "19635" uid: 298dfe28-960f-4e8c-b8c0-9e70216361e6 kind: Event lastTimestamp: "2026-05-20T15:51:44Z" message: 'Created pod: raw-sklearn-runtime-77323-predictor-6d8b8f8c49-g92hs' metadata: creationTimestamp: "2026-05-20T15:51:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:51:44Z" name: raw-sklearn-runtime-77323-predictor-6d8b8f8c49.18b150ec918f16d0 namespace: kserve-ci-e2e-test resourceVersion: "19647" uid: 5c413dc0-52a7-44f8-800e-35f6d77087be reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:51:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-77323-predictor namespace: kserve-ci-e2e-test resourceVersion: "19633" uid: 9ad48c91-bb12-43cf-86e7-695c9793505d kind: Event lastTimestamp: "2026-05-20T15:51:44Z" message: Scaled up replica set raw-sklearn-runtime-77323-predictor-6d8b8f8c49 from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:51:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:51:44Z" name: raw-sklearn-runtime-77323-predictor.18b150ec8e254f83 namespace: kserve-ci-e2e-test resourceVersion: "19637" uid: ac7b070c-d433-4e34-a1bf-d62c597502b0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:51:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-77323-predictor namespace: kserve-ci-e2e-test resourceVersion: "19634" uid: 02f18e03-a0bd-4dbd-9b1c-e07ad1e73fbc kind: Event lastTimestamp: "2026-05-20T15:52:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:51:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:52:29Z" name: raw-sklearn-runtime-77323-predictor.18b150f00cb7044a namespace: kserve-ci-e2e-test resourceVersion: "20007" uid: 589ea594-9271-4622-b8db-49d0f52a538d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:51:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-77323-predictor namespace: kserve-ci-e2e-test resourceVersion: "19634" uid: 02f18e03-a0bd-4dbd-9b1c-e07ad1e73fbc kind: Event lastTimestamp: "2026-05-20T15:52:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:51:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:52:29Z" name: raw-sklearn-runtime-77323-predictor.18b150f00d20cac5 namespace: kserve-ci-e2e-test resourceVersion: "20008" uid: dbc31d6e-aaa1-4022-9442-45bbb6d1e1fe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:52:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-77323-predictor namespace: kserve-ci-e2e-test resourceVersion: "19796" uid: 02f18e03-a0bd-4dbd-9b1c-e07ad1e73fbc kind: Event lastTimestamp: "2026-05-20T15:52:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:52:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:52:59Z" name: raw-sklearn-runtime-77323-predictor.18b150fa8a32b66e namespace: kserve-ci-e2e-test resourceVersion: "20235" uid: a247bed7-45d0-418a-bc8e-14d17ad4fb3c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:52:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-77323-predictor namespace: kserve-ci-e2e-test resourceVersion: "19796" uid: 02f18e03-a0bd-4dbd-9b1c-e07ad1e73fbc kind: Event lastTimestamp: "2026-05-20T15:52:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:52:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:52:59Z" name: raw-sklearn-runtime-77323-predictor.18b150fa8a9cd8bc namespace: kserve-ci-e2e-test resourceVersion: "20236" uid: eb205ad3-4d57-4286-ad83-cbf93f3cfe35 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-05-20T15:51:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-77323 namespace: kserve-ci-e2e-test resourceVersion: "19605" uid: fac2dbcf-16d8-4451-b3d4-8d58d006d398 kind: Event lastTimestamp: "2026-05-20T15:52:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:51:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:52:56Z" name: raw-sklearn-runtime-77323.18b150ec8e026506 namespace: kserve-ci-e2e-test resourceVersion: "20210" uid: 4739394a-b8e8-4b23-9277-3ed3d8ac94e5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:51:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-77323 namespace: kserve-ci-e2e-test resourceVersion: "19605" uid: fac2dbcf-16d8-4451-b3d4-8d58d006d398 kind: Event lastTimestamp: "2026-05-20T15:51:44Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-77323": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-77323": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:51:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:51:44Z" name: raw-sklearn-runtime-77323.18b150ec8e82f01a namespace: kserve-ci-e2e-test resourceVersion: "19639" uid: ce79a8d9-115c-41fa-b27d-5c1ada841546 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:52:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-77323 namespace: kserve-ci-e2e-test resourceVersion: "20208" uid: fac2dbcf-16d8-4451-b3d4-8d58d006d398 kind: Event lastTimestamp: "2026-05-20T15:52:56Z" message: InferenceService [raw-sklearn-runtime-77323] is Ready metadata: creationTimestamp: "2026-05-20T15:52:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:52:56Z" name: raw-sklearn-runtime-77323.18b150fd34cf99ae namespace: kserve-ci-e2e-test resourceVersion: "20209" uid: 5f46d368-a77f-49a8-8d2f-65f556b785bf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "20507"