--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:35:08.140716Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-687035-predictor-857cbdf89c-qnws7 namespace: kserve-ci-e2e-test resourceVersion: "22920" uid: df050d88-56c8-49d8-b981-4aaf0edfc7cf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-687035-predictor-857cbdf89c-qnws7 to ip-10-0-134-36.ec2.internal metadata: creationTimestamp: "2026-04-28T19:35:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:35:08Z" name: isvc-init-fail-687035-predictor-857cbdf89c-qnws7.18aa9c58cbc37f2c namespace: kserve-ci-e2e-test resourceVersion: "22926" uid: fdbe157d-f0f6-4755-9385-aaf33a1a38a4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f8cb4f8bc-qf7hx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:08Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-687035-predictor-857cbdf89c-qnws7 namespace: kserve-ci-e2e-test resourceVersion: "22924" uid: df050d88-56c8-49d8-b981-4aaf0edfc7cf kind: Event lastTimestamp: "2026-04-28T19:35:08Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-init-fail-687035-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-28T19:35:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:35:08Z" name: isvc-init-fail-687035-predictor-857cbdf89c-qnws7.18aa9c58d9f3c214 namespace: kserve-ci-e2e-test resourceVersion: "22938" uid: 35e4f40e-2c38-4032-a126-3e06527f9d8d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:09Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-687035-predictor-857cbdf89c-qnws7 namespace: kserve-ci-e2e-test resourceVersion: "22930" uid: df050d88-56c8-49d8-b981-4aaf0edfc7cf kind: Event lastTimestamp: "2026-04-28T19:35:09Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:35:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:35:09Z" name: isvc-init-fail-687035-predictor-857cbdf89c-qnws7.18aa9c5908caf167 namespace: kserve-ci-e2e-test resourceVersion: "22948" uid: eabb6e84-c73c-42f5-a822-f9fbc9a863aa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:35:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-687035-predictor-857cbdf89c-qnws7 namespace: kserve-ci-e2e-test resourceVersion: "22924" uid: df050d88-56c8-49d8-b981-4aaf0edfc7cf kind: Event lastTimestamp: "2026-04-28T19:35:13Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3460f014e7dc0a9d3daafe0716ca9eadf865f2892e0a5103d0b876da9f34891f" already present on machine metadata: creationTimestamp: "2026-04-28T19:35:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:35:13Z" name: isvc-init-fail-687035-predictor-857cbdf89c-qnws7.18aa9c590a2079ce namespace: kserve-ci-e2e-test resourceVersion: "22995" uid: b00c7b59-ec95-4511-b73b-eeea16932fc7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:35:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-687035-predictor-857cbdf89c-qnws7 namespace: kserve-ci-e2e-test resourceVersion: "22924" uid: df050d88-56c8-49d8-b981-4aaf0edfc7cf kind: Event lastTimestamp: "2026-04-28T19:35:13Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-28T19:35:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:35:13Z" name: isvc-init-fail-687035-predictor-857cbdf89c-qnws7.18aa9c590d94d7ad namespace: kserve-ci-e2e-test resourceVersion: "22997" uid: c439de86-ca21-4169-9a55-b606bc59653d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:35:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-687035-predictor-857cbdf89c-qnws7 namespace: kserve-ci-e2e-test resourceVersion: "22924" uid: df050d88-56c8-49d8-b981-4aaf0edfc7cf kind: Event lastTimestamp: "2026-04-28T19:35:13Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-28T19:35:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:35:13Z" name: isvc-init-fail-687035-predictor-857cbdf89c-qnws7.18aa9c590e1a8f02 namespace: kserve-ci-e2e-test resourceVersion: "23000" uid: 6fc9b58c-e583-4266-acb2-22c1a3038806 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-687035-predictor-857cbdf89c-qnws7 namespace: kserve-ci-e2e-test resourceVersion: "22924" uid: df050d88-56c8-49d8-b981-4aaf0edfc7cf kind: Event lastTimestamp: "2026-04-28T19:35:17Z" message: Back-off restarting failed container storage-initializer in pod isvc-init-fail-687035-predictor-857cbdf89c-qnws7_kserve-ci-e2e-test(df050d88-56c8-49d8-b981-4aaf0edfc7cf) metadata: creationTimestamp: "2026-04-28T19:35:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:35:17Z" name: isvc-init-fail-687035-predictor-857cbdf89c-qnws7.18aa9c5af1780c26 namespace: kserve-ci-e2e-test resourceVersion: "23031" uid: 6038921c-0694-444f-9aac-7670b817f677 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-687035-predictor-857cbdf89c namespace: kserve-ci-e2e-test resourceVersion: "22910" uid: 08c1f41c-d0f5-49f1-81ee-f5d8054367e0 kind: Event lastTimestamp: "2026-04-28T19:35:08Z" message: 'Created pod: isvc-init-fail-687035-predictor-857cbdf89c-qnws7' metadata: creationTimestamp: "2026-04-28T19:35:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:35:08Z" name: isvc-init-fail-687035-predictor-857cbdf89c.18aa9c58cb588ea6 namespace: kserve-ci-e2e-test resourceVersion: "22923" uid: 7e0c27a1-2d55-420d-beef-cea1296d809e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-687035-predictor namespace: kserve-ci-e2e-test resourceVersion: "22907" uid: 0b020a6f-b816-464f-925e-4ad315cbfaa7 kind: Event lastTimestamp: "2026-04-28T19:35:08Z" message: Scaled up replica set isvc-init-fail-687035-predictor-857cbdf89c from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:35:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:35:08Z" name: isvc-init-fail-687035-predictor.18aa9c58c89a94c9 namespace: kserve-ci-e2e-test resourceVersion: "22912" uid: 76155f3a-2ba6-4fbb-9696-f37695ad88b4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-28T19:35:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-687035 namespace: kserve-ci-e2e-test resourceVersion: "22880" uid: 03311970-e170-4a8c-9e77-5942101cca54 kind: Event lastTimestamp: "2026-04-28T19:35:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:35:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:35:17Z" name: isvc-init-fail-687035.18aa9c58c8a09aa2 namespace: kserve-ci-e2e-test resourceVersion: "23035" uid: 9b0ceffb-7318-4915-8a7e-c0c73dc3667d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-687035 namespace: kserve-ci-e2e-test resourceVersion: "22880" uid: 03311970-e170-4a8c-9e77-5942101cca54 kind: Event lastTimestamp: "2026-04-28T19:35:08Z" message: 'Failed to update status for InferenceService "isvc-init-fail-687035": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-687035": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:35:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:35:08Z" name: isvc-init-fail-687035.18aa9c58c9428ef4 namespace: kserve-ci-e2e-test resourceVersion: "22917" uid: c99c860e-e4f6-43dc-af68-98db72481f52 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-687035 namespace: kserve-ci-e2e-test resourceVersion: "22880" uid: 03311970-e170-4a8c-9e77-5942101cca54 kind: Event lastTimestamp: "2026-04-28T19:35:08Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-687035": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:35:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:35:08Z" name: isvc-init-fail-687035.18aa9c58c942e87e namespace: kserve-ci-e2e-test resourceVersion: "22918" uid: a40dbaae-8a2c-4626-988c-12172497fcbd reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:29:47.171430Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp namespace: kserve-ci-e2e-test resourceVersion: "18744" uid: 0926c6f8-5cea-4436-a832-15b9b25b565c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp to ip-10-0-134-36.ec2.internal metadata: creationTimestamp: "2026-04-28T19:29:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:29:47Z" name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp.18aa9c0e1080e1ac namespace: kserve-ci-e2e-test resourceVersion: "18748" uid: 099c2c47-9f88-41f5-a978-b1de44a3936b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f8cb4f8bc-qf7hx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:47Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp namespace: kserve-ci-e2e-test resourceVersion: "18752" uid: 0926c6f8-5cea-4436-a832-15b9b25b565c kind: Event lastTimestamp: "2026-04-28T19:29:47Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:29:47Z" name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp.18aa9c0e29acac89 namespace: kserve-ci-e2e-test resourceVersion: "18762" uid: 06b0de25-8249-4f05-a73d-4b8a64623160 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp namespace: kserve-ci-e2e-test resourceVersion: "18746" uid: 0926c6f8-5cea-4436-a832-15b9b25b565c kind: Event lastTimestamp: "2026-04-28T19:29:47Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3460f014e7dc0a9d3daafe0716ca9eadf865f2892e0a5103d0b876da9f34891f" already present on machine metadata: creationTimestamp: "2026-04-28T19:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:47Z" name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp.18aa9c0e2b51db13 namespace: kserve-ci-e2e-test resourceVersion: "18764" uid: 834a815f-4c7d-460d-a20c-ab0d3122e066 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp namespace: kserve-ci-e2e-test resourceVersion: "18746" uid: 0926c6f8-5cea-4436-a832-15b9b25b565c kind: Event lastTimestamp: "2026-04-28T19:29:47Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-28T19:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:47Z" name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp.18aa9c0e2e76a358 namespace: kserve-ci-e2e-test resourceVersion: "18765" uid: 31583de0-5915-442a-ab8b-c70ce1a526f6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp namespace: kserve-ci-e2e-test resourceVersion: "18746" uid: 0926c6f8-5cea-4436-a832-15b9b25b565c kind: Event lastTimestamp: "2026-04-28T19:29:47Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-28T19:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:47Z" name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp.18aa9c0e2eeb6382 namespace: kserve-ci-e2e-test resourceVersion: "18766" uid: 7bf901ee-7801-4af1-a727-28fc868a2171 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp namespace: kserve-ci-e2e-test resourceVersion: "18746" uid: 0926c6f8-5cea-4436-a832-15b9b25b565c kind: Event lastTimestamp: "2026-04-28T19:29:51Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-28T19:29:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:51Z" name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp.18aa9c0f0e6ade31 namespace: kserve-ci-e2e-test resourceVersion: "18827" uid: c318b41a-5bb3-483e-8d8e-d1c9d69c71d0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp namespace: kserve-ci-e2e-test resourceVersion: "18746" uid: 0926c6f8-5cea-4436-a832-15b9b25b565c kind: Event lastTimestamp: "2026-04-28T19:29:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:29:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:51Z" name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp.18aa9c0f12ee27d0 namespace: kserve-ci-e2e-test resourceVersion: "18832" uid: b72f8c97-4f22-49fd-85c5-a130925ee768 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp namespace: kserve-ci-e2e-test resourceVersion: "18746" uid: 0926c6f8-5cea-4436-a832-15b9b25b565c kind: Event lastTimestamp: "2026-04-28T19:29:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:29:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:51Z" name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp.18aa9c0f136219a1 namespace: kserve-ci-e2e-test resourceVersion: "18833" uid: d1859b6e-51c5-4a67-a973-e9b6d78c7fe4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp namespace: kserve-ci-e2e-test resourceVersion: "18746" uid: 0926c6f8-5cea-4436-a832-15b9b25b565c kind: Event lastTimestamp: "2026-04-28T19:29:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:29:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:51Z" name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp.18aa9c0f136f0de3 namespace: kserve-ci-e2e-test resourceVersion: "18834" uid: 9b15665a-8809-45af-9c1c-6938f2446f4b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp namespace: kserve-ci-e2e-test resourceVersion: "18746" uid: 0926c6f8-5cea-4436-a832-15b9b25b565c kind: Event lastTimestamp: "2026-04-28T19:29:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:29:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:51Z" name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp.18aa9c0f17f6705b namespace: kserve-ci-e2e-test resourceVersion: "18835" uid: 75c871d5-0c04-4151-90b3-fc85c1856e4a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp namespace: kserve-ci-e2e-test resourceVersion: "18746" uid: 0926c6f8-5cea-4436-a832-15b9b25b565c kind: Event lastTimestamp: "2026-04-28T19:29:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:29:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:51Z" name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp.18aa9c0f186df692 namespace: kserve-ci-e2e-test resourceVersion: "18836" uid: 9d9a8469-04d8-48dc-aaa2-8cb2afc4226b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp namespace: kserve-ci-e2e-test resourceVersion: "18746" uid: 0926c6f8-5cea-4436-a832-15b9b25b565c kind: Event lastTimestamp: "2026-04-28T19:29:51Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:de59d4f440abaeb1e71b5977a2145cdbe8db88ded8ac16ca09f179d82ba41738" already present on machine metadata: creationTimestamp: "2026-04-28T19:29:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:51Z" name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp.18aa9c0f187ab2c5 namespace: kserve-ci-e2e-test resourceVersion: "18837" uid: 8d1da1ae-8576-423b-882b-379bdddf3eff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp namespace: kserve-ci-e2e-test resourceVersion: "18746" uid: 0926c6f8-5cea-4436-a832-15b9b25b565c kind: Event lastTimestamp: "2026-04-28T19:29:51Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-28T19:29:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:51Z" name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp.18aa9c0f1b1cd3ed namespace: kserve-ci-e2e-test resourceVersion: "18838" uid: 96e97d5a-64df-4ee8-8dfe-75de858f1538 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp namespace: kserve-ci-e2e-test resourceVersion: "18746" uid: 0926c6f8-5cea-4436-a832-15b9b25b565c kind: Event lastTimestamp: "2026-04-28T19:29:51Z" message: Started container agent metadata: creationTimestamp: "2026-04-28T19:29:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:51Z" name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp.18aa9c0f1bd4c171 namespace: kserve-ci-e2e-test resourceVersion: "18839" uid: 2844149d-0577-4f88-920c-a2fac1e61d18 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-28T19:29:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp namespace: kserve-ci-e2e-test resourceVersion: "18746" uid: 0926c6f8-5cea-4436-a832-15b9b25b565c kind: Event lastTimestamp: "2026-04-28T19:31:28Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:29:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:28Z" name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp.18aa9c0f4a6434c4 namespace: kserve-ci-e2e-test resourceVersion: "20311" uid: 0ea8fe7b-31d2-4c76-951b-bf81d9c527be reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-28T19:29:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp namespace: kserve-ci-e2e-test resourceVersion: "18746" uid: 0926c6f8-5cea-4436-a832-15b9b25b565c kind: Event lastTimestamp: "2026-04-28T19:31:28Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-28T19:29:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:28Z" name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp.18aa9c0f4a6d2bfd namespace: kserve-ci-e2e-test resourceVersion: "20312" uid: 11c47adb-cba6-4d18-a559-3271b64fb101 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp namespace: kserve-ci-e2e-test resourceVersion: "18746" uid: 0926c6f8-5cea-4436-a832-15b9b25b565c kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:12Z" name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp.18aa9c21da977acd namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 227d276b-c748-48cc-9256-ee4ee92150de reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp namespace: kserve-ci-e2e-test resourceVersion: "18746" uid: 0926c6f8-5cea-4436-a832-15b9b25b565c kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:12Z" name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp.18aa9c21da979cbd namespace: kserve-ci-e2e-test resourceVersion: "20034" uid: 0aa0bd96-3764-4d54-9493-3d956bc6d0bf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp namespace: kserve-ci-e2e-test resourceVersion: "18746" uid: 0926c6f8-5cea-4436-a832-15b9b25b565c kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:12Z" name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp.18aa9c21da98187a namespace: kserve-ci-e2e-test resourceVersion: "20035" uid: 490278c1-db5a-48b8-bc65-72d57e0c8905 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-28T19:31:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp namespace: kserve-ci-e2e-test resourceVersion: "18746" uid: 0926c6f8-5cea-4436-a832-15b9b25b565c kind: Event lastTimestamp: "2026-04-28T19:31:33Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:31:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:33Z" name: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp.18aa9c22268c9239 namespace: kserve-ci-e2e-test resourceVersion: "20382" uid: 869e8843-9b57-458a-9582-a87570ec2870 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-a8bff-predictor-b8fd65b67 namespace: kserve-ci-e2e-test resourceVersion: "18729" uid: 7252ea4f-f032-4354-9562-cc2a6d4aa86d kind: Event lastTimestamp: "2026-04-28T19:29:47Z" message: 'Created pod: isvc-logger-raw-a8bff-predictor-b8fd65b67-48kfp' metadata: creationTimestamp: "2026-04-28T19:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:29:47Z" name: isvc-logger-raw-a8bff-predictor-b8fd65b67.18aa9c0e1010d25a namespace: kserve-ci-e2e-test resourceVersion: "18745" uid: 3b05828e-ab1d-4d2f-9b95-375f759c643b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-a8bff-predictor namespace: kserve-ci-e2e-test resourceVersion: "18727" uid: e49122b4-3d01-44ed-9ca9-31398480dcd3 kind: Event lastTimestamp: "2026-04-28T19:29:46Z" message: Scaled up replica set isvc-logger-raw-a8bff-predictor-b8fd65b67 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:29:46Z" name: isvc-logger-raw-a8bff-predictor.18aa9c0dedb4ff20 namespace: kserve-ci-e2e-test resourceVersion: "18731" uid: 53ac902c-1100-431b-b859-d570f34e95ab reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:30:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-a8bff-predictor namespace: kserve-ci-e2e-test resourceVersion: "18728" uid: 1afc61c3-2c86-4aef-9242-c944a5041bf9 kind: Event lastTimestamp: "2026-04-28T19:30:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:30:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:30:16Z" name: isvc-logger-raw-a8bff-predictor.18aa9c116c2e62ab namespace: kserve-ci-e2e-test resourceVersion: "19204" uid: 37f6776e-e7a0-419c-84b6-b40b5cd38a08 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:30:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-a8bff-predictor namespace: kserve-ci-e2e-test resourceVersion: "18728" uid: 1afc61c3-2c86-4aef-9242-c944a5041bf9 kind: Event lastTimestamp: "2026-04-28T19:30:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:30:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:30:16Z" name: isvc-logger-raw-a8bff-predictor.18aa9c116c961bb1 namespace: kserve-ci-e2e-test resourceVersion: "19206" uid: e7d3a7d1-0859-4e81-8a61-6df448ce8b2b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:30:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-a8bff-predictor namespace: kserve-ci-e2e-test resourceVersion: "19001" uid: 1afc61c3-2c86-4aef-9242-c944a5041bf9 kind: Event lastTimestamp: "2026-04-28T19:31:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:30:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:31:01Z" name: isvc-logger-raw-a8bff-predictor.18aa9c1869ed8084 namespace: kserve-ci-e2e-test resourceVersion: "19828" uid: 539e4c34-7872-4c44-bb16-d99ec523083c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:30:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-a8bff-predictor namespace: kserve-ci-e2e-test resourceVersion: "19001" uid: 1afc61c3-2c86-4aef-9242-c944a5041bf9 kind: Event lastTimestamp: "2026-04-28T19:31:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:30:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:31:01Z" name: isvc-logger-raw-a8bff-predictor.18aa9c186a6ec888 namespace: kserve-ci-e2e-test resourceVersion: "19830" uid: 1c28cbe1-931e-4895-942c-ef89e4f3bb05 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 25 eventTime: null firstTimestamp: "2026-04-28T19:29:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-a8bff namespace: kserve-ci-e2e-test resourceVersion: "18717" uid: a7e3f75e-2988-4d08-b2cc-93ca29fe26cb kind: Event lastTimestamp: "2026-04-28T19:30:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:30:02Z" name: isvc-logger-raw-a8bff.18aa9c0ded8e2e08 namespace: kserve-ci-e2e-test resourceVersion: "19016" uid: 71b98598-c3ad-4f21-b6c9-7399f059fbe2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:30:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-a8bff namespace: kserve-ci-e2e-test resourceVersion: "19793" uid: a7e3f75e-2988-4d08-b2cc-93ca29fe26cb kind: Event lastTimestamp: "2026-04-28T19:30:58Z" message: InferenceService [isvc-logger-raw-a8bff] is Ready metadata: creationTimestamp: "2026-04-28T19:30:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:30:59Z" name: isvc-logger-raw-a8bff.18aa9c1ec6005ecc namespace: kserve-ci-e2e-test resourceVersion: "19794" uid: ad18983b-22a5-4c60-945b-7337e92d09ef reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:33:32.620488Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n namespace: kserve-ci-e2e-test resourceVersion: "22063" uid: 1ef18fbd-984f-4923-a03a-bdf19cc4cd9a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n to ip-10-0-134-36.ec2.internal metadata: creationTimestamp: "2026-04-28T19:33:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:33:32Z" name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n.18aa9c428e5082f1 namespace: kserve-ci-e2e-test resourceVersion: "22067" uid: 4a90d619-89c8-46d2-84c5-e9b25e0ed73d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f8cb4f8bc-qf7hx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:33:33Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n namespace: kserve-ci-e2e-test resourceVersion: "22071" uid: 1ef18fbd-984f-4923-a03a-bdf19cc4cd9a kind: Event lastTimestamp: "2026-04-28T19:33:33Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:33:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:33:33Z" name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n.18aa9c42a7505c32 namespace: kserve-ci-e2e-test resourceVersion: "22079" uid: 253c07c3-3415-4451-998b-f1510cab2ea5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:33:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: 1ef18fbd-984f-4923-a03a-bdf19cc4cd9a kind: Event lastTimestamp: "2026-04-28T19:33:33Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3460f014e7dc0a9d3daafe0716ca9eadf865f2892e0a5103d0b876da9f34891f" already present on machine metadata: creationTimestamp: "2026-04-28T19:33:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:33:33Z" name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n.18aa9c42a897b3fd namespace: kserve-ci-e2e-test resourceVersion: "22081" uid: dd080b1f-d54c-4aa1-8976-e50a6f50ecdb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:33:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: 1ef18fbd-984f-4923-a03a-bdf19cc4cd9a kind: Event lastTimestamp: "2026-04-28T19:33:33Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-28T19:33:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:33:33Z" name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n.18aa9c42ab119e5a namespace: kserve-ci-e2e-test resourceVersion: "22082" uid: 4f74cf62-196e-4b23-8aeb-d36ad39af15f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:33:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: 1ef18fbd-984f-4923-a03a-bdf19cc4cd9a kind: Event lastTimestamp: "2026-04-28T19:33:33Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-28T19:33:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:33:33Z" name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n.18aa9c42ab83608c namespace: kserve-ci-e2e-test resourceVersion: "22083" uid: fa4a607c-e768-48d0-ae2a-5187563f06d7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:33:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: 1ef18fbd-984f-4923-a03a-bdf19cc4cd9a kind: Event lastTimestamp: "2026-04-28T19:33:37Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-28T19:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:33:37Z" name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n.18aa9c4396a26643 namespace: kserve-ci-e2e-test resourceVersion: "22118" uid: 6bff0b60-f9b8-46e7-8522-841be6d5361f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:33:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: 1ef18fbd-984f-4923-a03a-bdf19cc4cd9a kind: Event lastTimestamp: "2026-04-28T19:33:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:33:37Z" name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n.18aa9c4399826a18 namespace: kserve-ci-e2e-test resourceVersion: "22120" uid: 344a9726-44a4-4b6d-af18-86fa6f2fdcde reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:33:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: 1ef18fbd-984f-4923-a03a-bdf19cc4cd9a kind: Event lastTimestamp: "2026-04-28T19:33:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:33:37Z" name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n.18aa9c439a031474 namespace: kserve-ci-e2e-test resourceVersion: "22123" uid: 5db4fe1b-fb78-4ea1-af7c-0374799688f8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:33:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: 1ef18fbd-984f-4923-a03a-bdf19cc4cd9a kind: Event lastTimestamp: "2026-04-28T19:33:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:33:37Z" name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n.18aa9c439a107d52 namespace: kserve-ci-e2e-test resourceVersion: "22124" uid: 03892c68-9881-4631-a4d3-4233b1009195 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:33:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: 1ef18fbd-984f-4923-a03a-bdf19cc4cd9a kind: Event lastTimestamp: "2026-04-28T19:33:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:33:37Z" name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n.18aa9c439c7d894f namespace: kserve-ci-e2e-test resourceVersion: "22126" uid: f8c45d09-066c-4772-883b-7d3280105679 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:33:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: 1ef18fbd-984f-4923-a03a-bdf19cc4cd9a kind: Event lastTimestamp: "2026-04-28T19:33:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:33:37Z" name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n.18aa9c439ce9a3ce namespace: kserve-ci-e2e-test resourceVersion: "22127" uid: 3c63f3db-2bc4-4111-a809-dda7955fa867 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-28T19:33:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: 1ef18fbd-984f-4923-a03a-bdf19cc4cd9a kind: Event lastTimestamp: "2026-04-28T19:34:35Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:33:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:34:35Z" name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n.18aa9c440e498121 namespace: kserve-ci-e2e-test resourceVersion: "22547" uid: a907a847-2837-4b09-ac63-b0fc1121d14c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: 1ef18fbd-984f-4923-a03a-bdf19cc4cd9a kind: Event lastTimestamp: "2026-04-28T19:35:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:35:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:35:08Z" name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n.18aa9c58c83f064c namespace: kserve-ci-e2e-test resourceVersion: "22911" uid: 2eca2041-b0d0-4926-b7b8-bfdddc18a3af reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: 1ef18fbd-984f-4923-a03a-bdf19cc4cd9a kind: Event lastTimestamp: "2026-04-28T19:35:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:35:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:35:08Z" name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n.18aa9c58c83fa3b3 namespace: kserve-ci-e2e-test resourceVersion: "22916" uid: ef5ac854-fa7f-4c72-a835-e80892abf253 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: 1ef18fbd-984f-4923-a03a-bdf19cc4cd9a kind: Event lastTimestamp: "2026-04-28T19:35:10Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:35:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:35:10Z" name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n.18aa9c593e6a5ac7 namespace: kserve-ci-e2e-test resourceVersion: "22966" uid: 89cd88e1-ad63-4339-a65f-5ea546924b40 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:33:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd namespace: kserve-ci-e2e-test resourceVersion: "22037" uid: 33ac7999-e9ff-4118-8b6a-743e5fdb7264 kind: Event lastTimestamp: "2026-04-28T19:33:32Z" message: 'Created pod: isvc-primary-b9e6a0-predictor-5bdbbd7cbd-68l4n' metadata: creationTimestamp: "2026-04-28T19:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:33:32Z" name: isvc-primary-b9e6a0-predictor-5bdbbd7cbd.18aa9c428de47344 namespace: kserve-ci-e2e-test resourceVersion: "22064" uid: 67205848-ffd7-45b0-9209-fe22eceac271 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:33:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-b9e6a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "22035" uid: 870ae93e-5386-43a8-b452-d7dfe425f44a kind: Event lastTimestamp: "2026-04-28T19:33:32Z" message: Scaled up replica set isvc-primary-b9e6a0-predictor-5bdbbd7cbd from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:33:32Z" name: isvc-primary-b9e6a0-predictor.18aa9c4274760739 namespace: kserve-ci-e2e-test resourceVersion: "22039" uid: 741ef895-b3e5-4ca2-8517-6c028ae425ae reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:33:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-b9e6a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "22036" uid: 905ba082-a660-405a-bb19-b7d740652055 kind: Event lastTimestamp: "2026-04-28T19:34:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:34:17Z" name: isvc-primary-b9e6a0-predictor.18aa9c45f35680b1 namespace: kserve-ci-e2e-test resourceVersion: "22416" uid: 1b329872-50db-41d1-9311-4e4b09e07c39 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:33:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-b9e6a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "22036" uid: 905ba082-a660-405a-bb19-b7d740652055 kind: Event lastTimestamp: "2026-04-28T19:34:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:33:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:34:17Z" name: isvc-primary-b9e6a0-predictor.18aa9c45f3c0e11a namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: 9800cb93-afda-46d2-a652-8dd36634fb26 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-28T19:34:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-b9e6a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "22207" uid: 905ba082-a660-405a-bb19-b7d740652055 kind: Event lastTimestamp: "2026-04-28T19:35:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:35:02Z" name: isvc-primary-b9e6a0-predictor.18aa9c497282d965 namespace: kserve-ci-e2e-test resourceVersion: "22810" uid: f2a7e3e4-0312-4a60-89a1-767fd6856564 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-28T19:34:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-b9e6a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "22207" uid: 905ba082-a660-405a-bb19-b7d740652055 kind: Event lastTimestamp: "2026-04-28T19:35:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:35:02Z" name: isvc-primary-b9e6a0-predictor.18aa9c4972e515b0 namespace: kserve-ci-e2e-test resourceVersion: "22812" uid: db8dd3d7-2f7f-4a76-b955-3f19515019cc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-28T19:33:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-b9e6a0 namespace: kserve-ci-e2e-test resourceVersion: "22025" uid: 82f8d8bf-21a6-4b2d-9822-3c37f4f8c936 kind: Event lastTimestamp: "2026-04-28T19:34:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:34:45Z" name: isvc-primary-b9e6a0.18aa9c42743fbcbe namespace: kserve-ci-e2e-test resourceVersion: "22624" uid: 87df6e3b-a799-4cd1-bded-ee05b4c223d4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:34:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-b9e6a0 namespace: kserve-ci-e2e-test resourceVersion: "22622" uid: 82f8d8bf-21a6-4b2d-9822-3c37f4f8c936 kind: Event lastTimestamp: "2026-04-28T19:34:45Z" message: InferenceService [isvc-primary-b9e6a0] is Ready metadata: creationTimestamp: "2026-04-28T19:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:34:45Z" name: isvc-primary-b9e6a0.18aa9c5370d863e4 namespace: kserve-ci-e2e-test resourceVersion: "22623" uid: 1ca9cf45-bdd7-41e6-888c-02613be661c5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:24:10.615818Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn namespace: kserve-ci-e2e-test resourceVersion: "13352" uid: f6149254-214c-471c-8ec1-826372d2bb1d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn to ip-10-0-134-36.ec2.internal metadata: creationTimestamp: "2026-04-28T19:24:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:24:10Z" name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn.18aa9bbfb439cedf namespace: kserve-ci-e2e-test resourceVersion: "13360" uid: c0f5d38b-eae6-4d48-834b-1bbcaf20a8c4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f8cb4f8bc-qf7hx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:24:11Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn namespace: kserve-ci-e2e-test resourceVersion: "13364" uid: f6149254-214c-471c-8ec1-826372d2bb1d kind: Event lastTimestamp: "2026-04-28T19:24:11Z" message: Add eth0 [10.134.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:24:11Z" name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn.18aa9bbfcd9f81d1 namespace: kserve-ci-e2e-test resourceVersion: "13382" uid: 36718edb-30ee-4a0f-837e-cadc268a7638 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:24:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn namespace: kserve-ci-e2e-test resourceVersion: "13358" uid: f6149254-214c-471c-8ec1-826372d2bb1d kind: Event lastTimestamp: "2026-04-28T19:24:11Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:3460f014e7dc0a9d3daafe0716ca9eadf865f2892e0a5103d0b876da9f34891f" metadata: creationTimestamp: "2026-04-28T19:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:24:11Z" name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn.18aa9bbfd036f4d9 namespace: kserve-ci-e2e-test resourceVersion: "13385" uid: bd8b5176-a839-4d57-8953-cedc965ee73c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:24:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn namespace: kserve-ci-e2e-test resourceVersion: "13358" uid: f6149254-214c-471c-8ec1-826372d2bb1d kind: Event lastTimestamp: "2026-04-28T19:24:14Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:3460f014e7dc0a9d3daafe0716ca9eadf865f2892e0a5103d0b876da9f34891f" in 3.442s (3.442s including waiting). Image size: 301485528 bytes.' metadata: creationTimestamp: "2026-04-28T19:24:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:24:14Z" name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn.18aa9bc09d66eed8 namespace: kserve-ci-e2e-test resourceVersion: "13439" uid: 94619b94-584f-482f-a562-d744ac933d5e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:24:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn namespace: kserve-ci-e2e-test resourceVersion: "13358" uid: f6149254-214c-471c-8ec1-826372d2bb1d kind: Event lastTimestamp: "2026-04-28T19:24:14Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-28T19:24:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:24:14Z" name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn.18aa9bc0a09153aa namespace: kserve-ci-e2e-test resourceVersion: "13441" uid: f4424475-1055-421b-b101-85caea28c0b0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:24:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn namespace: kserve-ci-e2e-test resourceVersion: "13358" uid: f6149254-214c-471c-8ec1-826372d2bb1d kind: Event lastTimestamp: "2026-04-28T19:24:14Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-28T19:24:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:24:14Z" name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn.18aa9bc0a102571e namespace: kserve-ci-e2e-test resourceVersion: "13442" uid: 9e6f5f5e-a32d-4aa8-ba3c-f5be08899325 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:24:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn namespace: kserve-ci-e2e-test resourceVersion: "13358" uid: f6149254-214c-471c-8ec1-826372d2bb1d kind: Event lastTimestamp: "2026-04-28T19:24:18Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1404" metadata: creationTimestamp: "2026-04-28T19:24:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:24:18Z" name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn.18aa9bc1847e348d namespace: kserve-ci-e2e-test resourceVersion: "13539" uid: fe3e7c9f-58bb-4b52-906c-14cb08d222f9 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:24:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn namespace: kserve-ci-e2e-test resourceVersion: "13358" uid: f6149254-214c-471c-8ec1-826372d2bb1d kind: Event lastTimestamp: "2026-04-28T19:24:31Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1404" in 13.194s (13.194s including waiting). Image size: 1560922562 bytes.' metadata: creationTimestamp: "2026-04-28T19:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:24:31Z" name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn.18aa9bc496ecbf3a namespace: kserve-ci-e2e-test resourceVersion: "13745" uid: 5d7e99da-b632-4952-ba69-1f84fdbfcd42 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:24:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn namespace: kserve-ci-e2e-test resourceVersion: "13358" uid: f6149254-214c-471c-8ec1-826372d2bb1d kind: Event lastTimestamp: "2026-04-28T19:24:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:24:31Z" name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn.18aa9bc49a4fd156 namespace: kserve-ci-e2e-test resourceVersion: "13747" uid: 0cf893b6-e878-46f0-bb5d-7933fe59c526 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:24:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn namespace: kserve-ci-e2e-test resourceVersion: "13358" uid: f6149254-214c-471c-8ec1-826372d2bb1d kind: Event lastTimestamp: "2026-04-28T19:24:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:24:31Z" name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn.18aa9bc49ac48b76 namespace: kserve-ci-e2e-test resourceVersion: "13748" uid: 4e956a6c-8a7e-45f8-bd34-920e778e57bf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:24:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn namespace: kserve-ci-e2e-test resourceVersion: "13358" uid: f6149254-214c-471c-8ec1-826372d2bb1d kind: Event lastTimestamp: "2026-04-28T19:24:31Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-28T19:24:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:24:31Z" name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn.18aa9bc49ad2d030 namespace: kserve-ci-e2e-test resourceVersion: "13749" uid: ec4b173d-c400-43bb-842f-3a1b88e9e0f8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:24:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn namespace: kserve-ci-e2e-test resourceVersion: "13358" uid: f6149254-214c-471c-8ec1-826372d2bb1d kind: Event lastTimestamp: "2026-04-28T19:24:33Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.307s (2.307s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-28T19:24:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:24:33Z" name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn.18aa9bc5245d42b2 namespace: kserve-ci-e2e-test resourceVersion: "13781" uid: 632c791d-0aad-4c2d-818d-e20377e3ac9c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:24:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn namespace: kserve-ci-e2e-test resourceVersion: "13358" uid: f6149254-214c-471c-8ec1-826372d2bb1d kind: Event lastTimestamp: "2026-04-28T19:24:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:24:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:24:34Z" name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn.18aa9bc5279dffbe namespace: kserve-ci-e2e-test resourceVersion: "13782" uid: d4252857-40d2-443d-9a95-e7b9456a992b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:24:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn namespace: kserve-ci-e2e-test resourceVersion: "13358" uid: f6149254-214c-471c-8ec1-826372d2bb1d kind: Event lastTimestamp: "2026-04-28T19:24:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:24:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:24:34Z" name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn.18aa9bc528104cb9 namespace: kserve-ci-e2e-test resourceVersion: "13783" uid: 518b7588-42b6-4154-8491-67ca3fafe0c7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:24:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn namespace: kserve-ci-e2e-test resourceVersion: "13358" uid: f6149254-214c-471c-8ec1-826372d2bb1d kind: Event lastTimestamp: "2026-04-28T19:24:34Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:de59d4f440abaeb1e71b5977a2145cdbe8db88ded8ac16ca09f179d82ba41738" metadata: creationTimestamp: "2026-04-28T19:24:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:24:34Z" name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn.18aa9bc5281d1fb4 namespace: kserve-ci-e2e-test resourceVersion: "13784" uid: 89cb7d67-27f1-4294-b14d-b6ca73ef9617 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:24:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn namespace: kserve-ci-e2e-test resourceVersion: "13358" uid: f6149254-214c-471c-8ec1-826372d2bb1d kind: Event lastTimestamp: "2026-04-28T19:24:36Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:de59d4f440abaeb1e71b5977a2145cdbe8db88ded8ac16ca09f179d82ba41738" in 2.357s (2.357s including waiting). Image size: 238051450 bytes.' metadata: creationTimestamp: "2026-04-28T19:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:24:36Z" name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn.18aa9bc5b4a24013 namespace: kserve-ci-e2e-test resourceVersion: "13816" uid: f1ae53e3-c575-47eb-a499-6237165a5c92 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:24:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn namespace: kserve-ci-e2e-test resourceVersion: "13358" uid: f6149254-214c-471c-8ec1-826372d2bb1d kind: Event lastTimestamp: "2026-04-28T19:24:36Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-28T19:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:24:36Z" name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn.18aa9bc5b7b404c3 namespace: kserve-ci-e2e-test resourceVersion: "13817" uid: 1dc36dd8-9741-43ad-9f26-1bc424fa1ea3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:24:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn namespace: kserve-ci-e2e-test resourceVersion: "13358" uid: f6149254-214c-471c-8ec1-826372d2bb1d kind: Event lastTimestamp: "2026-04-28T19:24:36Z" message: Started container agent metadata: creationTimestamp: "2026-04-28T19:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:24:36Z" name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn.18aa9bc5b83c968c namespace: kserve-ci-e2e-test resourceVersion: "13819" uid: d0797421-a1be-4dc5-9656-b3e21406ccf7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-28T19:24:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn namespace: kserve-ci-e2e-test resourceVersion: "13358" uid: f6149254-214c-471c-8ec1-826372d2bb1d kind: Event lastTimestamp: "2026-04-28T19:26:13Z" message: 'Readiness probe failed: dial tcp 10.134.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:24:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:26:13Z" name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn.18aa9bc5f525eb72 namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: 2abb1bc9-4291-4d94-8d50-321a870c4754 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-28T19:24:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn namespace: kserve-ci-e2e-test resourceVersion: "13358" uid: f6149254-214c-471c-8ec1-826372d2bb1d kind: Event lastTimestamp: "2026-04-28T19:26:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-28T19:24:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:26:13Z" name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn.18aa9bc5f5304323 namespace: kserve-ci-e2e-test resourceVersion: "15327" uid: 0d74a436-12bc-4ad8-b117-81ad1d57b66b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:25:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn namespace: kserve-ci-e2e-test resourceVersion: "13358" uid: f6149254-214c-471c-8ec1-826372d2bb1d kind: Event lastTimestamp: "2026-04-28T19:25:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:25:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:25:55Z" name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn.18aa9bd82a5f76d2 namespace: kserve-ci-e2e-test resourceVersion: "14945" uid: c6f67f97-91c9-4077-b087-a1bcc946c668 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:25:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn namespace: kserve-ci-e2e-test resourceVersion: "13358" uid: f6149254-214c-471c-8ec1-826372d2bb1d kind: Event lastTimestamp: "2026-04-28T19:25:55Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-28T19:25:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:25:55Z" name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn.18aa9bd82a5fcb35 namespace: kserve-ci-e2e-test resourceVersion: "14948" uid: 445deaec-0503-4a21-adc3-1405056cde52 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:25:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn namespace: kserve-ci-e2e-test resourceVersion: "13358" uid: f6149254-214c-471c-8ec1-826372d2bb1d kind: Event lastTimestamp: "2026-04-28T19:25:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:25:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:25:55Z" name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn.18aa9bd82a5ffa4d namespace: kserve-ci-e2e-test resourceVersion: "14947" uid: 7746e59d-8ffb-4fcb-9bb4-a1dc5059e38e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-28T19:25:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn namespace: kserve-ci-e2e-test resourceVersion: "13358" uid: f6149254-214c-471c-8ec1-826372d2bb1d kind: Event lastTimestamp: "2026-04-28T19:26:18Z" message: 'Readiness probe failed: Get "https://10.134.0.17:8643/healthz": dial tcp 10.134.0.17:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:25:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:26:18Z" name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn.18aa9bd8d1381c0a namespace: kserve-ci-e2e-test resourceVersion: "15393" uid: 854449bc-3a87-4176-9043-603658fff9ea reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:24:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f namespace: kserve-ci-e2e-test resourceVersion: "13344" uid: 751b2482-3ef6-4111-b4e5-9dcac4b03043 kind: Event lastTimestamp: "2026-04-28T19:24:10Z" message: 'Created pod: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f-t74pn' metadata: creationTimestamp: "2026-04-28T19:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:24:10Z" name: isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f.18aa9bbfb371427d namespace: kserve-ci-e2e-test resourceVersion: "13355" uid: 3d4e483a-009d-4df5-8188-880090607a62 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:24:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-199b4-predictor namespace: kserve-ci-e2e-test resourceVersion: "13342" uid: 08d40b25-5b0c-4d74-9186-1c9b0a1a5ccc kind: Event lastTimestamp: "2026-04-28T19:24:10Z" message: Scaled up replica set isvc-raw-sklearn-batcher-199b4-predictor-7cdd66cc9f from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:24:10Z" name: isvc-raw-sklearn-batcher-199b4-predictor.18aa9bbfad722687 namespace: kserve-ci-e2e-test resourceVersion: "13345" uid: e6658322-33ef-4acb-bfd5-aab3e76e628f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-28T19:24:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-199b4-predictor namespace: kserve-ci-e2e-test resourceVersion: "13354" uid: 83fdcd98-49b1-4aa4-b43a-69e742176980 kind: Event lastTimestamp: "2026-04-28T19:25:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:24:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:25:10Z" name: isvc-raw-sklearn-batcher-199b4-predictor.18aa9bc333073a35 namespace: kserve-ci-e2e-test resourceVersion: "14302" uid: 5730f19f-8be3-44ef-b524-32303c49b910 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-28T19:24:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-199b4-predictor namespace: kserve-ci-e2e-test resourceVersion: "13354" uid: 83fdcd98-49b1-4aa4-b43a-69e742176980 kind: Event lastTimestamp: "2026-04-28T19:25:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:24:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:25:10Z" name: isvc-raw-sklearn-batcher-199b4-predictor.18aa9bc3337a7d13 namespace: kserve-ci-e2e-test resourceVersion: "14303" uid: da50754f-a5d6-48d1-98f0-71d22424cded reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:25:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-199b4-predictor namespace: kserve-ci-e2e-test resourceVersion: "13640" uid: 83fdcd98-49b1-4aa4-b43a-69e742176980 kind: Event lastTimestamp: "2026-04-28T19:25:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:25:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:25:40Z" name: isvc-raw-sklearn-batcher-199b4-predictor.18aa9bd130a7873e namespace: kserve-ci-e2e-test resourceVersion: "14717" uid: 9d648b17-ca28-4958-9a26-59c560fea5ef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:25:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-199b4-predictor namespace: kserve-ci-e2e-test resourceVersion: "13640" uid: 83fdcd98-49b1-4aa4-b43a-69e742176980 kind: Event lastTimestamp: "2026-04-28T19:25:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:25:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:25:40Z" name: isvc-raw-sklearn-batcher-199b4-predictor.18aa9bd131382386 namespace: kserve-ci-e2e-test resourceVersion: "14718" uid: 021ea945-9496-4088-9e6a-1a0bfc8d811a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 23 eventTime: null firstTimestamp: "2026-04-28T19:24:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-199b4 namespace: kserve-ci-e2e-test resourceVersion: "13330" uid: edcafba3-1e6c-4a87-88ea-53e817044428 kind: Event lastTimestamp: "2026-04-28T19:24:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:24:17Z" name: isvc-raw-sklearn-batcher-199b4.18aa9bbfb3c4fd52 namespace: kserve-ci-e2e-test resourceVersion: "13526" uid: 05883349-19b6-429a-887f-a0b111314c91 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:24:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-199b4 namespace: kserve-ci-e2e-test resourceVersion: "13330" uid: edcafba3-1e6c-4a87-88ea-53e817044428 kind: Event lastTimestamp: "2026-04-28T19:24:10Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-199b4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-199b4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:24:10Z" name: isvc-raw-sklearn-batcher-199b4.18aa9bbfbafc8460 namespace: kserve-ci-e2e-test resourceVersion: "13366" uid: f7179ff9-8d58-4cf3-a380-0fba03019dc5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:24:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-199b4 namespace: kserve-ci-e2e-test resourceVersion: "13330" uid: edcafba3-1e6c-4a87-88ea-53e817044428 kind: Event lastTimestamp: "2026-04-28T19:24:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-199b4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:24:10Z" name: isvc-raw-sklearn-batcher-199b4.18aa9bbfbafcd39f namespace: kserve-ci-e2e-test resourceVersion: "13368" uid: f6e6b2c8-0784-4777-b498-554dce63eb9e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:25:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-199b4 namespace: kserve-ci-e2e-test resourceVersion: "14763" uid: edcafba3-1e6c-4a87-88ea-53e817044428 kind: Event lastTimestamp: "2026-04-28T19:25:43Z" message: InferenceService [isvc-raw-sklearn-batcher-199b4] is Ready metadata: creationTimestamp: "2026-04-28T19:25:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:25:43Z" name: isvc-raw-sklearn-batcher-199b4.18aa9bd56700501a namespace: kserve-ci-e2e-test resourceVersion: "14765" uid: add9f6f2-a762-4c70-b915-a332a3e2bd49 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:34:51.957846Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-b9e6a0-predictor-5d7d9858d7-lh2hg namespace: kserve-ci-e2e-test resourceVersion: "22703" uid: a43f12ab-b64a-4dee-aea6-defd4e555c99 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-b9e6a0-predictor-5d7d9858d7-lh2hg to ip-10-0-134-36.ec2.internal metadata: creationTimestamp: "2026-04-28T19:34:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:34:51Z" name: isvc-secondary-b9e6a0-predictor-5d7d9858d7-lh2hg.18aa9c55073081fb namespace: kserve-ci-e2e-test resourceVersion: "22707" uid: 171217cc-e90c-4832-8e07-489d57ab1bea reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f8cb4f8bc-qf7hx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:34:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-b9e6a0-predictor-5d7d9858d7-lh2hg namespace: kserve-ci-e2e-test resourceVersion: "22705" uid: a43f12ab-b64a-4dee-aea6-defd4e555c99 kind: Event lastTimestamp: "2026-04-28T19:34:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-secondary-b9e6a0-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-28T19:34:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:34:52Z" name: isvc-secondary-b9e6a0-predictor-5d7d9858d7-lh2hg.18aa9c55125db59c namespace: kserve-ci-e2e-test resourceVersion: "22718" uid: 11ebee50-e0fc-4b08-9e46-2f953fc46253 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:34:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-b9e6a0-predictor-5d7d9858d7-lh2hg namespace: kserve-ci-e2e-test resourceVersion: "22711" uid: a43f12ab-b64a-4dee-aea6-defd4e555c99 kind: Event lastTimestamp: "2026-04-28T19:34:52Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:34:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:34:52Z" name: isvc-secondary-b9e6a0-predictor-5d7d9858d7-lh2hg.18aa9c5543ca3285 namespace: kserve-ci-e2e-test resourceVersion: "22729" uid: 5c349036-6684-4a53-9524-4395e427bdd2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:34:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-b9e6a0-predictor-5d7d9858d7-lh2hg namespace: kserve-ci-e2e-test resourceVersion: "22705" uid: a43f12ab-b64a-4dee-aea6-defd4e555c99 kind: Event lastTimestamp: "2026-04-28T19:34:57Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3460f014e7dc0a9d3daafe0716ca9eadf865f2892e0a5103d0b876da9f34891f" already present on machine metadata: creationTimestamp: "2026-04-28T19:34:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:34:57Z" name: isvc-secondary-b9e6a0-predictor-5d7d9858d7-lh2hg.18aa9c55451ef13a namespace: kserve-ci-e2e-test resourceVersion: "22764" uid: e5a3fc59-2a47-4a8e-a564-4d90197d67c4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:34:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-b9e6a0-predictor-5d7d9858d7-lh2hg namespace: kserve-ci-e2e-test resourceVersion: "22705" uid: a43f12ab-b64a-4dee-aea6-defd4e555c99 kind: Event lastTimestamp: "2026-04-28T19:34:57Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-28T19:34:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:34:57Z" name: isvc-secondary-b9e6a0-predictor-5d7d9858d7-lh2hg.18aa9c55487493e2 namespace: kserve-ci-e2e-test resourceVersion: "22769" uid: 2a6bf935-3853-4f2c-889f-9def2723f89b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:34:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-b9e6a0-predictor-5d7d9858d7-lh2hg namespace: kserve-ci-e2e-test resourceVersion: "22705" uid: a43f12ab-b64a-4dee-aea6-defd4e555c99 kind: Event lastTimestamp: "2026-04-28T19:34:57Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-28T19:34:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:34:57Z" name: isvc-secondary-b9e6a0-predictor-5d7d9858d7-lh2hg.18aa9c5548e78fd5 namespace: kserve-ci-e2e-test resourceVersion: "22770" uid: f683f717-4b0d-4039-9686-9ae0a522f299 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-b9e6a0-predictor-5d7d9858d7-lh2hg namespace: kserve-ci-e2e-test resourceVersion: "22705" uid: a43f12ab-b64a-4dee-aea6-defd4e555c99 kind: Event lastTimestamp: "2026-04-28T19:35:02Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-b9e6a0-predictor-5d7d9858d7-lh2hg_kserve-ci-e2e-test(a43f12ab-b64a-4dee-aea6-defd4e555c99) metadata: creationTimestamp: "2026-04-28T19:35:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:35:02Z" name: isvc-secondary-b9e6a0-predictor-5d7d9858d7-lh2hg.18aa9c57702ff6a8 namespace: kserve-ci-e2e-test resourceVersion: "22811" uid: acff9c38-bceb-4173-9f15-4fe58abf1a12 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:34:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-b9e6a0-predictor-5d7d9858d7 namespace: kserve-ci-e2e-test resourceVersion: "22694" uid: 62bc4583-16c1-4b3a-9150-9549945a1887 kind: Event lastTimestamp: "2026-04-28T19:34:51Z" message: 'Created pod: isvc-secondary-b9e6a0-predictor-5d7d9858d7-lh2hg' metadata: creationTimestamp: "2026-04-28T19:34:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:34:51Z" name: isvc-secondary-b9e6a0-predictor-5d7d9858d7.18aa9c5506b1cf80 namespace: kserve-ci-e2e-test resourceVersion: "22704" uid: 0fec761b-8c9a-4925-96e0-a3bbead9da9e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:34:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-b9e6a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "22692" uid: 95a35934-4b92-4664-b4a0-00f2a59ffa1b kind: Event lastTimestamp: "2026-04-28T19:34:51Z" message: Scaled up replica set isvc-secondary-b9e6a0-predictor-5d7d9858d7 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:34:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:34:51Z" name: isvc-secondary-b9e6a0-predictor.18aa9c550349a919 namespace: kserve-ci-e2e-test resourceVersion: "22696" uid: 3df5fdfb-d611-46d8-a216-a5bc806bede4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-b9e6a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "22693" uid: 41549cb8-ee3c-4257-9ce3-a6a0e2d6d9f5 kind: Event lastTimestamp: "2026-04-28T19:35:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:35:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:35:06Z" name: isvc-secondary-b9e6a0-predictor.18aa9c58822af32d namespace: kserve-ci-e2e-test resourceVersion: "22848" uid: 3357ff05-c96d-4a21-9a07-d981356b7ddb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-b9e6a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "22693" uid: 41549cb8-ee3c-4257-9ce3-a6a0e2d6d9f5 kind: Event lastTimestamp: "2026-04-28T19:35:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:35:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:35:06Z" name: isvc-secondary-b9e6a0-predictor.18aa9c588294decc namespace: kserve-ci-e2e-test resourceVersion: "22850" uid: 49f62a0e-f742-48e4-90f0-6cb3a060e400 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 14 eventTime: null firstTimestamp: "2026-04-28T19:34:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-b9e6a0 namespace: kserve-ci-e2e-test resourceVersion: "22681" uid: b1fcd56f-0446-4dc2-b086-f062eb66babf kind: Event lastTimestamp: "2026-04-28T19:35:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:34:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:35:03Z" name: isvc-secondary-b9e6a0.18aa9c550320cf51 namespace: kserve-ci-e2e-test resourceVersion: "22826" uid: 4f272151-9a5e-4531-b426-5f6c02ef1fc2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:34:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-b9e6a0 namespace: kserve-ci-e2e-test resourceVersion: "22681" uid: b1fcd56f-0446-4dc2-b086-f062eb66babf kind: Event lastTimestamp: "2026-04-28T19:34:51Z" message: 'Failed to update status for InferenceService "isvc-secondary-b9e6a0": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-b9e6a0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:34:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:34:51Z" name: isvc-secondary-b9e6a0.18aa9c550372c3fd namespace: kserve-ci-e2e-test resourceVersion: "22715" uid: 0a7f9116-9907-41f6-9d07-ed281ff1b19c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:34:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-b9e6a0 namespace: kserve-ci-e2e-test resourceVersion: "22681" uid: b1fcd56f-0446-4dc2-b086-f062eb66babf kind: Event lastTimestamp: "2026-04-28T19:34:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-b9e6a0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:34:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:34:51Z" name: isvc-secondary-b9e6a0.18aa9c55037336c1 namespace: kserve-ci-e2e-test resourceVersion: "22716" uid: cdd63607-1351-41cf-8e15-d50135dce8ae reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:25:56.481891Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf namespace: kserve-ci-e2e-test resourceVersion: "14979" uid: da2c2459-c227-459c-ae3e-9783f5fa460b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf to ip-10-0-134-36.ec2.internal metadata: creationTimestamp: "2026-04-28T19:25:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:25:56Z" name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf.18aa9bd85a55d277 namespace: kserve-ci-e2e-test resourceVersion: "14984" uid: ee13652d-2a93-43fd-aee7-b0ef65c3506d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f8cb4f8bc-qf7hx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:25:56Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf namespace: kserve-ci-e2e-test resourceVersion: "14988" uid: da2c2459-c227-459c-ae3e-9783f5fa460b kind: Event lastTimestamp: "2026-04-28T19:25:56Z" message: Add eth0 [10.134.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:25:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:25:56Z" name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf.18aa9bd873b8f3d8 namespace: kserve-ci-e2e-test resourceVersion: "15013" uid: 15ef03cc-4062-41cc-bf41-b9131865f2a2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:25:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf namespace: kserve-ci-e2e-test resourceVersion: "14981" uid: da2c2459-c227-459c-ae3e-9783f5fa460b kind: Event lastTimestamp: "2026-04-28T19:25:56Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3460f014e7dc0a9d3daafe0716ca9eadf865f2892e0a5103d0b876da9f34891f" already present on machine metadata: creationTimestamp: "2026-04-28T19:25:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:25:56Z" name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf.18aa9bd8758ea74f namespace: kserve-ci-e2e-test resourceVersion: "15015" uid: 3c9fa884-73f4-44be-8494-385b492fca8d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:25:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf namespace: kserve-ci-e2e-test resourceVersion: "14981" uid: da2c2459-c227-459c-ae3e-9783f5fa460b kind: Event lastTimestamp: "2026-04-28T19:25:56Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-28T19:25:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:25:56Z" name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf.18aa9bd878980f22 namespace: kserve-ci-e2e-test resourceVersion: "15016" uid: a32d1ebf-d9f5-4479-a3e1-9eafa36eb58c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:25:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf namespace: kserve-ci-e2e-test resourceVersion: "14981" uid: da2c2459-c227-459c-ae3e-9783f5fa460b kind: Event lastTimestamp: "2026-04-28T19:25:56Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-28T19:25:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:25:56Z" name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf.18aa9bd8790e893f namespace: kserve-ci-e2e-test resourceVersion: "15017" uid: 2e55fe0f-2860-490c-abe9-8fdb44772efd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:26:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf namespace: kserve-ci-e2e-test resourceVersion: "14981" uid: da2c2459-c227-459c-ae3e-9783f5fa460b kind: Event lastTimestamp: "2026-04-28T19:26:01Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-28T19:26:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:26:01Z" name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf.18aa9bd992870ecc namespace: kserve-ci-e2e-test resourceVersion: "15116" uid: b137f286-b7cf-4a3b-9f9b-cdab649b5553 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:26:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf namespace: kserve-ci-e2e-test resourceVersion: "14981" uid: da2c2459-c227-459c-ae3e-9783f5fa460b kind: Event lastTimestamp: "2026-04-28T19:26:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:26:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:26:01Z" name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf.18aa9bd995619889 namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: fdb54167-b9d7-4e46-9ce1-e2b9d52060a0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:26:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf namespace: kserve-ci-e2e-test resourceVersion: "14981" uid: da2c2459-c227-459c-ae3e-9783f5fa460b kind: Event lastTimestamp: "2026-04-28T19:26:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:26:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:26:01Z" name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf.18aa9bd995d37211 namespace: kserve-ci-e2e-test resourceVersion: "15122" uid: c28f6b6f-1fc1-40ee-92bf-3fd55c0dda22 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:26:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf namespace: kserve-ci-e2e-test resourceVersion: "14981" uid: da2c2459-c227-459c-ae3e-9783f5fa460b kind: Event lastTimestamp: "2026-04-28T19:26:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:26:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:26:01Z" name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf.18aa9bd995e0b75b namespace: kserve-ci-e2e-test resourceVersion: "15124" uid: 76890920-b7a1-4cfa-a040-0cfda9e4c0b6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:26:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf namespace: kserve-ci-e2e-test resourceVersion: "14981" uid: da2c2459-c227-459c-ae3e-9783f5fa460b kind: Event lastTimestamp: "2026-04-28T19:26:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:26:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:26:01Z" name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf.18aa9bd998e83430 namespace: kserve-ci-e2e-test resourceVersion: "15125" uid: bc88a8c3-b070-4d63-8047-17db64b22b66 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:26:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf namespace: kserve-ci-e2e-test resourceVersion: "14981" uid: da2c2459-c227-459c-ae3e-9783f5fa460b kind: Event lastTimestamp: "2026-04-28T19:26:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:26:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:26:01Z" name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf.18aa9bd9996b8b0b namespace: kserve-ci-e2e-test resourceVersion: "15126" uid: f11b55c5-b056-4948-9a43-82980a21e0da reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-28T19:26:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf namespace: kserve-ci-e2e-test resourceVersion: "14981" uid: da2c2459-c227-459c-ae3e-9783f5fa460b kind: Event lastTimestamp: "2026-04-28T19:27:48Z" message: 'Readiness probe failed: dial tcp 10.134.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:26:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:48Z" name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf.18aa9bd9ce9ea755 namespace: kserve-ci-e2e-test resourceVersion: "16843" uid: 13b2e8d5-d70b-4008-bd21-7482a6cd2526 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf namespace: kserve-ci-e2e-test resourceVersion: "14981" uid: da2c2459-c227-459c-ae3e-9783f5fa460b kind: Event lastTimestamp: "2026-04-28T19:27:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:47Z" name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf.18aa9bf225e62bb9 namespace: kserve-ci-e2e-test resourceVersion: "16792" uid: 5cbcbcf7-0097-4c38-b972-45f7c9b425b3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf namespace: kserve-ci-e2e-test resourceVersion: "14981" uid: da2c2459-c227-459c-ae3e-9783f5fa460b kind: Event lastTimestamp: "2026-04-28T19:27:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:47Z" name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf.18aa9bf225ec0406 namespace: kserve-ci-e2e-test resourceVersion: "16793" uid: 4eccbf0b-baa5-43ea-9bb1-daf069d031fb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf namespace: kserve-ci-e2e-test resourceVersion: "14981" uid: da2c2459-c227-459c-ae3e-9783f5fa460b kind: Event lastTimestamp: "2026-04-28T19:27:48Z" message: 'Readiness probe failed: Get "https://10.134.0.18:8643/healthz": dial tcp 10.134.0.18:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:27:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:48Z" name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf.18aa9bf27cf5dc63 namespace: kserve-ci-e2e-test resourceVersion: "16842" uid: 6238bfde-25d9-49a1-b2bc-b2cc2113d26a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:25:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5 namespace: kserve-ci-e2e-test resourceVersion: "14964" uid: 39cc3a21-2da3-4786-8f0c-84464801b881 kind: Event lastTimestamp: "2026-04-28T19:25:56Z" message: 'Created pod: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5-9mmsf' metadata: creationTimestamp: "2026-04-28T19:25:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:25:56Z" name: isvc-sklearn-graph-raw-357ae-predictor-56864bdff5.18aa9bd8599a08d7 namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: fd2720b6-ba28-4ac3-b778-eb37c2a707bf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:25:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-357ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "14961" uid: 999e343f-f8e2-4d61-b308-1813e5a234fb kind: Event lastTimestamp: "2026-04-28T19:25:56Z" message: Scaled up replica set isvc-sklearn-graph-raw-357ae-predictor-56864bdff5 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:25:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:25:56Z" name: isvc-sklearn-graph-raw-357ae-predictor.18aa9bd83dfcef88 namespace: kserve-ci-e2e-test resourceVersion: "14967" uid: e823261e-2021-41b4-9350-3e18f6480879 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:26:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-357ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: eb1c83ff-ebe7-4c53-9b9b-08dcac4ab94e kind: Event lastTimestamp: "2026-04-28T19:26:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:26:26Z" name: isvc-sklearn-graph-raw-357ae-predictor.18aa9bdbbc70abd3 namespace: kserve-ci-e2e-test resourceVersion: "15506" uid: ca9b8287-4097-438e-b635-b08a5f497c19 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:26:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-357ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: eb1c83ff-ebe7-4c53-9b9b-08dcac4ab94e kind: Event lastTimestamp: "2026-04-28T19:26:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:26:26Z" name: isvc-sklearn-graph-raw-357ae-predictor.18aa9bdbbce8683f namespace: kserve-ci-e2e-test resourceVersion: "15507" uid: e6692a08-43ec-4cf9-b982-02c8dfd074c9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-28T19:26:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-357ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: eb1c83ff-ebe7-4c53-9b9b-08dcac4ab94e kind: Event lastTimestamp: "2026-04-28T19:27:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:27:41Z" name: isvc-sklearn-graph-raw-357ae-predictor.18aa9be2baf8306a namespace: kserve-ci-e2e-test resourceVersion: "16604" uid: 3e0f9e54-76d8-4eb2-9607-df7404d016d0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-28T19:26:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-357ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: eb1c83ff-ebe7-4c53-9b9b-08dcac4ab94e kind: Event lastTimestamp: "2026-04-28T19:27:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:27:41Z" name: isvc-sklearn-graph-raw-357ae-predictor.18aa9be2bb5d2cf9 namespace: kserve-ci-e2e-test resourceVersion: "16605" uid: ea16a3e8-78d4-432c-b455-775db39818ee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 24 eventTime: null firstTimestamp: "2026-04-28T19:25:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-357ae namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: d69e099c-5bd5-463a-8fd4-bd836649a87c kind: Event lastTimestamp: "2026-04-28T19:26:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:25:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:26:12Z" name: isvc-sklearn-graph-raw-357ae.18aa9bd83ce2c2f1 namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: 99e5bf38-5193-4f48-acd3-5e74bd24037d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:25:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-357ae namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: d69e099c-5bd5-463a-8fd4-bd836649a87c kind: Event lastTimestamp: "2026-04-28T19:25:56Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-357ae": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-357ae": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:25:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:25:56Z" name: isvc-sklearn-graph-raw-357ae.18aa9bd83f715328 namespace: kserve-ci-e2e-test resourceVersion: "14970" uid: 21640986-0f2a-47a9-9460-bcb987141811 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-357ae namespace: kserve-ci-e2e-test resourceVersion: "16265" uid: d69e099c-5bd5-463a-8fd4-bd836649a87c kind: Event lastTimestamp: "2026-04-28T19:27:19Z" message: InferenceService [isvc-sklearn-graph-raw-357ae] is Ready metadata: creationTimestamp: "2026-04-28T19:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:27:19Z" name: isvc-sklearn-graph-raw-357ae.18aa9beba4a550fd namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: f91cb445-a844-4429-b8e0-fb497a35fed3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:27:46.870408Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf namespace: kserve-ci-e2e-test resourceVersion: "16746" uid: 6b4c02de-688d-4548-98fc-22809c409d68 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf to ip-10-0-134-36.ec2.internal metadata: creationTimestamp: "2026-04-28T19:27:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:27:46Z" name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf.18aa9bf20e00f677 namespace: kserve-ci-e2e-test resourceVersion: "16750" uid: 4af49ceb-0273-4899-9485-343d08e0660f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f8cb4f8bc-qf7hx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:47Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf namespace: kserve-ci-e2e-test resourceVersion: "16755" uid: 6b4c02de-688d-4548-98fc-22809c409d68 kind: Event lastTimestamp: "2026-04-28T19:27:47Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:27:47Z" name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf.18aa9bf227c50e45 namespace: kserve-ci-e2e-test resourceVersion: "16795" uid: fb91e2c7-28fb-4971-94c7-b190bdcaabfe reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 6b4c02de-688d-4548-98fc-22809c409d68 kind: Event lastTimestamp: "2026-04-28T19:27:47Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3460f014e7dc0a9d3daafe0716ca9eadf865f2892e0a5103d0b876da9f34891f" already present on machine metadata: creationTimestamp: "2026-04-28T19:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:47Z" name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf.18aa9bf22c518bfb namespace: kserve-ci-e2e-test resourceVersion: "16797" uid: 7ddc398c-1a5b-4ba0-9313-590846b04faa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 6b4c02de-688d-4548-98fc-22809c409d68 kind: Event lastTimestamp: "2026-04-28T19:27:47Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-28T19:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:47Z" name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf.18aa9bf22f28e3d6 namespace: kserve-ci-e2e-test resourceVersion: "16798" uid: 73cf1159-6c75-4337-b2e4-c49b04a07faf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 6b4c02de-688d-4548-98fc-22809c409d68 kind: Event lastTimestamp: "2026-04-28T19:27:47Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-28T19:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:47Z" name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf.18aa9bf22f95b4a3 namespace: kserve-ci-e2e-test resourceVersion: "16799" uid: 2eccc40a-61e2-4b94-aee8-dabe9f5286ab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 6b4c02de-688d-4548-98fc-22809c409d68 kind: Event lastTimestamp: "2026-04-28T19:27:51Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-28T19:27:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:51Z" name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf.18aa9bf3079b33f0 namespace: kserve-ci-e2e-test resourceVersion: "16887" uid: fcc1e394-a397-4952-a5da-9819c3bf28b9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 6b4c02de-688d-4548-98fc-22809c409d68 kind: Event lastTimestamp: "2026-04-28T19:27:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:27:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:51Z" name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf.18aa9bf30c504d81 namespace: kserve-ci-e2e-test resourceVersion: "16892" uid: 255d52d6-1008-43b9-ac24-a9be0550519b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 6b4c02de-688d-4548-98fc-22809c409d68 kind: Event lastTimestamp: "2026-04-28T19:27:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:27:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:51Z" name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf.18aa9bf30cc04522 namespace: kserve-ci-e2e-test resourceVersion: "16893" uid: df26370e-6333-466e-82de-224b591fbdea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 6b4c02de-688d-4548-98fc-22809c409d68 kind: Event lastTimestamp: "2026-04-28T19:27:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:27:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:51Z" name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf.18aa9bf30cccfe9e namespace: kserve-ci-e2e-test resourceVersion: "16894" uid: 29c48a35-b82a-41b3-821d-8c3e2e74e318 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 6b4c02de-688d-4548-98fc-22809c409d68 kind: Event lastTimestamp: "2026-04-28T19:27:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:27:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:51Z" name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf.18aa9bf30ffac62b namespace: kserve-ci-e2e-test resourceVersion: "16898" uid: 1b1b31ca-a058-4cf9-9c7a-c37371a0b962 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 6b4c02de-688d-4548-98fc-22809c409d68 kind: Event lastTimestamp: "2026-04-28T19:27:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:27:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:51Z" name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf.18aa9bf3106d321f namespace: kserve-ci-e2e-test resourceVersion: "16900" uid: ebf3fc40-3614-4345-b07c-5a4de145e9e0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-28T19:27:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 6b4c02de-688d-4548-98fc-22809c409d68 kind: Event lastTimestamp: "2026-04-28T19:28:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:27:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:28:58Z" name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf.18aa9bf343a1592a namespace: kserve-ci-e2e-test resourceVersion: "17880" uid: 49f6f738-414c-42f0-908c-9c72d297267e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 6b4c02de-688d-4548-98fc-22809c409d68 kind: Event lastTimestamp: "2026-04-28T19:29:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:26Z" name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf.18aa9c095bd2d2ab namespace: kserve-ci-e2e-test resourceVersion: "18388" uid: e1270ffb-39f9-4496-b570-d6678c74fc0e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 6b4c02de-688d-4548-98fc-22809c409d68 kind: Event lastTimestamp: "2026-04-28T19:29:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:26Z" name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf.18aa9c095bd3695c namespace: kserve-ci-e2e-test resourceVersion: "18389" uid: 529fbaf0-926f-4a23-bd40-05793e8983a9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 6b4c02de-688d-4548-98fc-22809c409d68 kind: Event lastTimestamp: "2026-04-28T19:29:28Z" message: 'Readiness probe failed: Get "https://10.134.0.20:8643/healthz": dial tcp 10.134.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:29:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:28Z" name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf.18aa9c099e5ca8b4 namespace: kserve-ci-e2e-test resourceVersion: "18420" uid: 0354ec0a-880f-4390-8577-3507a6181a4b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff namespace: kserve-ci-e2e-test resourceVersion: "16731" uid: 96b79209-9860-4c73-8c8c-09dd646658d4 kind: Event lastTimestamp: "2026-04-28T19:27:46Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff-t6hnf' metadata: creationTimestamp: "2026-04-28T19:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:27:46Z" name: isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff.18aa9bf20da183e4 namespace: kserve-ci-e2e-test resourceVersion: "16747" uid: cd772d7d-aeb1-4739-b55d-588654291b7b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-5628a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16729" uid: 12df1d33-4632-42b5-b4b0-093ab5aff3dd kind: Event lastTimestamp: "2026-04-28T19:27:46Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-5628a-predictor-c488b8fff from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:27:46Z" name: isvc-sklearn-graph-raw-hpa-5628a-predictor.18aa9bf1f6f85bf7 namespace: kserve-ci-e2e-test resourceVersion: "16732" uid: 662db9e9-d205-47bc-9b4e-33b9040cedfc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:28:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-5628a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16730" uid: 65d3a3ab-59d6-4d50-9a71-d1a7e50caf14 kind: Event lastTimestamp: "2026-04-28T19:28:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:28:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:28:31Z" name: isvc-sklearn-graph-raw-hpa-5628a-predictor.18aa9bf5756f3486 namespace: kserve-ci-e2e-test resourceVersion: "17513" uid: 8f29bb74-2d77-4bc7-85f3-9fad30b28818 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:28:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-5628a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16730" uid: 65d3a3ab-59d6-4d50-9a71-d1a7e50caf14 kind: Event lastTimestamp: "2026-04-28T19:28:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:28:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:28:31Z" name: isvc-sklearn-graph-raw-hpa-5628a-predictor.18aa9bf575d280d0 namespace: kserve-ci-e2e-test resourceVersion: "17514" uid: bfdd9e59-af3b-41da-a37a-f2c6ecb383f7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-28T19:28:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-5628a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17080" uid: 65d3a3ab-59d6-4d50-9a71-d1a7e50caf14 kind: Event lastTimestamp: "2026-04-28T19:29:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:28:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:29:16Z" name: isvc-sklearn-graph-raw-hpa-5628a-predictor.18aa9bf8f50c9dfc namespace: kserve-ci-e2e-test resourceVersion: "18169" uid: f444e9c4-b91c-44d4-bfb7-aec699de214e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-28T19:28:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-5628a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17080" uid: 65d3a3ab-59d6-4d50-9a71-d1a7e50caf14 kind: Event lastTimestamp: "2026-04-28T19:29:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:28:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:29:16Z" name: isvc-sklearn-graph-raw-hpa-5628a-predictor.18aa9bf8f5718e7a namespace: kserve-ci-e2e-test resourceVersion: "18170" uid: 716d71e8-e9f4-4334-86c6-7eff18344430 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 25 eventTime: null firstTimestamp: "2026-04-28T19:27:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-5628a namespace: kserve-ci-e2e-test resourceVersion: "16718" uid: d8d5ade9-167e-4650-8457-aad429a72c45 kind: Event lastTimestamp: "2026-04-28T19:28:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:28:03Z" name: isvc-sklearn-graph-raw-hpa-5628a.18aa9bf1f6cd16d9 namespace: kserve-ci-e2e-test resourceVersion: "17120" uid: d28fb580-c78e-4e1d-b816-5798c928d851 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-5628a namespace: kserve-ci-e2e-test resourceVersion: "17952" uid: d8d5ade9-167e-4650-8457-aad429a72c45 kind: Event lastTimestamp: "2026-04-28T19:29:02Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-5628a] is Ready metadata: creationTimestamp: "2026-04-28T19:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:29:02Z" name: isvc-sklearn-graph-raw-hpa-5628a.18aa9c0393e96d38 namespace: kserve-ci-e2e-test resourceVersion: "17955" uid: c116334b-4e6a-4a1a-82b6-e2aa7ebffb6d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:31:12.379396Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7 namespace: kserve-ci-e2e-test resourceVersion: "20039" uid: 9d14f466-716d-4651-92f3-f764554541ae kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7 to ip-10-0-134-36.ec2.internal metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:31:12Z" name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7.18aa9c21e74b3e17 namespace: kserve-ci-e2e-test resourceVersion: "20043" uid: 1768b5e9-0868-42fe-b649-0ef0b14d4cec reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f8cb4f8bc-qf7hx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7 namespace: kserve-ci-e2e-test resourceVersion: "20047" uid: 9d14f466-716d-4651-92f3-f764554541ae kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:31:12Z" name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7.18aa9c21ffe8e214 namespace: kserve-ci-e2e-test resourceVersion: "20056" uid: dcdde742-eb7a-46b1-a572-59b536b2612d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7 namespace: kserve-ci-e2e-test resourceVersion: "20040" uid: 9d14f466-716d-4651-92f3-f764554541ae kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3460f014e7dc0a9d3daafe0716ca9eadf865f2892e0a5103d0b876da9f34891f" already present on machine metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:12Z" name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7.18aa9c220189e840 namespace: kserve-ci-e2e-test resourceVersion: "20058" uid: c7b2c6a5-914e-434b-8cf5-946a1ad1163e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7 namespace: kserve-ci-e2e-test resourceVersion: "20040" uid: 9d14f466-716d-4651-92f3-f764554541ae kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:12Z" name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7.18aa9c2204f8d1f3 namespace: kserve-ci-e2e-test resourceVersion: "20059" uid: 96ea83dc-faf0-4a85-a02b-2463bf013f9f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7 namespace: kserve-ci-e2e-test resourceVersion: "20040" uid: 9d14f466-716d-4651-92f3-f764554541ae kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:12Z" name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7.18aa9c22056b12e6 namespace: kserve-ci-e2e-test resourceVersion: "20060" uid: 4891ec0d-3eff-4842-9941-4422cdd714ff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7 namespace: kserve-ci-e2e-test resourceVersion: "20040" uid: 9d14f466-716d-4651-92f3-f764554541ae kind: Event lastTimestamp: "2026-04-28T19:31:16Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-28T19:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:16Z" name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7.18aa9c22e776948d namespace: kserve-ci-e2e-test resourceVersion: "20126" uid: 68afd5aa-7b84-4470-a6e1-73e4bb16f0e8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7 namespace: kserve-ci-e2e-test resourceVersion: "20040" uid: 9d14f466-716d-4651-92f3-f764554541ae kind: Event lastTimestamp: "2026-04-28T19:31:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:16Z" name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7.18aa9c22ea75afd4 namespace: kserve-ci-e2e-test resourceVersion: "20128" uid: ea67d37e-82ce-4c3c-829c-694e5130433a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7 namespace: kserve-ci-e2e-test resourceVersion: "20040" uid: 9d14f466-716d-4651-92f3-f764554541ae kind: Event lastTimestamp: "2026-04-28T19:31:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:16Z" name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7.18aa9c22eae6ab82 namespace: kserve-ci-e2e-test resourceVersion: "20129" uid: 041c7a4b-4228-4feb-bc38-e595cfe5b45b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7 namespace: kserve-ci-e2e-test resourceVersion: "20040" uid: 9d14f466-716d-4651-92f3-f764554541ae kind: Event lastTimestamp: "2026-04-28T19:31:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:16Z" name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7.18aa9c22eaf27969 namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 619ec339-aad8-46bc-b6ee-51a98f9ecab3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7 namespace: kserve-ci-e2e-test resourceVersion: "20040" uid: 9d14f466-716d-4651-92f3-f764554541ae kind: Event lastTimestamp: "2026-04-28T19:31:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:16Z" name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7.18aa9c22ed7fda72 namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: d74d0a0a-5423-408e-840d-ea7917a7e380 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7 namespace: kserve-ci-e2e-test resourceVersion: "20040" uid: 9d14f466-716d-4651-92f3-f764554541ae kind: Event lastTimestamp: "2026-04-28T19:31:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:16Z" name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7.18aa9c22edf74397 namespace: kserve-ci-e2e-test resourceVersion: "20135" uid: 1e345913-3c43-4cd9-aeb4-9212c1ea0f91 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 16 eventTime: null firstTimestamp: "2026-04-28T19:31:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7 namespace: kserve-ci-e2e-test resourceVersion: "20040" uid: 9d14f466-716d-4651-92f3-f764554541ae kind: Event lastTimestamp: "2026-04-28T19:33:37Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:31:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:33:37Z" name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7.18aa9c235f42ff46 namespace: kserve-ci-e2e-test resourceVersion: "22134" uid: ec8b5600-8d6d-4b42-a952-ec866ff37e95 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:33:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7 namespace: kserve-ci-e2e-test resourceVersion: "20040" uid: 9d14f466-716d-4651-92f3-f764554541ae kind: Event lastTimestamp: "2026-04-28T19:33:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:33:32Z" name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7.18aa9c427b8040c3 namespace: kserve-ci-e2e-test resourceVersion: "22058" uid: c38e10fb-5716-4e57-a568-f17d232887e9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:33:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7 namespace: kserve-ci-e2e-test resourceVersion: "20040" uid: 9d14f466-716d-4651-92f3-f764554541ae kind: Event lastTimestamp: "2026-04-28T19:33:32Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:33:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:33:32Z" name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7.18aa9c427b80df44 namespace: kserve-ci-e2e-test resourceVersion: "22059" uid: f8762ba1-24e5-45c5-b43c-52854706db98 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:33:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7 namespace: kserve-ci-e2e-test resourceVersion: "20040" uid: 9d14f466-716d-4651-92f3-f764554541ae kind: Event lastTimestamp: "2026-04-28T19:33:39Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:33:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:33:39Z" name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7.18aa9c4309936299 namespace: kserve-ci-e2e-test resourceVersion: "22147" uid: 82019054-5c3a-46cc-b903-37264f448091 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f namespace: kserve-ci-e2e-test resourceVersion: "20007" uid: 11c8c3a9-d263-47ca-a89b-816e453a4e99 kind: Event lastTimestamp: "2026-04-28T19:31:12Z" message: 'Created pod: isvc-sklearn-scale-raw-9745c-predictor-74db4886f-jb4q7' metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:31:12Z" name: isvc-sklearn-scale-raw-9745c-predictor-74db4886f.18aa9c21e6c28cf9 namespace: kserve-ci-e2e-test resourceVersion: "20041" uid: e97c9975-dc1b-4832-ba5c-7c50270c71c5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-9745c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20005" uid: 7b23a2bc-9c9c-43ef-a81b-7019e32d882f kind: Event lastTimestamp: "2026-04-28T19:31:11Z" message: Scaled up replica set isvc-sklearn-scale-raw-9745c-predictor-74db4886f from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:31:11Z" name: isvc-sklearn-scale-raw-9745c-predictor.18aa9c21d066aba8 namespace: kserve-ci-e2e-test resourceVersion: "20008" uid: fb3392f4-c378-4f89-9900-3fa00d207b15 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:31:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-9745c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20006" uid: 9c0caf62-2a2b-46d6-a05b-a5d40154f0af kind: Event lastTimestamp: "2026-04-28T19:31:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:31:57Z" name: isvc-sklearn-scale-raw-9745c-predictor.18aa9c254e9ec0b5 namespace: kserve-ci-e2e-test resourceVersion: "20708" uid: e13ee865-3f97-4556-b8ca-40c165c811dd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:31:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-9745c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20006" uid: 9c0caf62-2a2b-46d6-a05b-a5d40154f0af kind: Event lastTimestamp: "2026-04-28T19:31:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:31:57Z" name: isvc-sklearn-scale-raw-9745c-predictor.18aa9c254f0057af namespace: kserve-ci-e2e-test resourceVersion: "20709" uid: b501b3cd-0125-4153-b0fb-b7e2c373fc36 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:32:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-9745c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20291" uid: 9c0caf62-2a2b-46d6-a05b-a5d40154f0af kind: Event lastTimestamp: "2026-04-28T19:33:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:32:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:33:27Z" name: isvc-sklearn-scale-raw-9745c-predictor.18aa9c2fcc1155fb namespace: kserve-ci-e2e-test resourceVersion: "21945" uid: 35253c59-8ca4-4e2f-b653-934629a50a86 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:32:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-9745c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20291" uid: 9c0caf62-2a2b-46d6-a05b-a5d40154f0af kind: Event lastTimestamp: "2026-04-28T19:33:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:32:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:33:27Z" name: isvc-sklearn-scale-raw-9745c-predictor.18aa9c2fcc6c5034 namespace: kserve-ci-e2e-test resourceVersion: "21946" uid: 2058b5d8-90ca-4345-b1ef-43c332d473ff reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 25 eventTime: null firstTimestamp: "2026-04-28T19:31:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-9745c namespace: kserve-ci-e2e-test resourceVersion: "19995" uid: 7d727401-e447-48f2-a5ca-7e0ef93ddefd kind: Event lastTimestamp: "2026-04-28T19:31:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:31:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:31:20Z" name: isvc-sklearn-scale-raw-9745c.18aa9c21cffe6e3f namespace: kserve-ci-e2e-test resourceVersion: "20205" uid: 1fd8f0f1-a688-411c-80df-cc995290cccd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:33:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-9745c namespace: kserve-ci-e2e-test resourceVersion: "21969" uid: 7d727401-e447-48f2-a5ca-7e0ef93ddefd kind: Event lastTimestamp: "2026-04-28T19:33:28Z" message: InferenceService [isvc-sklearn-scale-raw-9745c] is Ready metadata: creationTimestamp: "2026-04-28T19:33:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:33:28Z" name: isvc-sklearn-scale-raw-9745c.18aa9c4185a9b231 namespace: kserve-ci-e2e-test resourceVersion: "21970" uid: fe90784c-a1f7-4cb5-8688-17803ab25055 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:25:57.032802Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: c348e295-37a9-4e1d-a806-e255f953dbbe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj to ip-10-0-134-36.ec2.internal metadata: creationTimestamp: "2026-04-28T19:25:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:25:57Z" name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj.18aa9bd87b2c23af namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: 6e58f139-814a-45b5-a762-6566e25564f1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f8cb4f8bc-qf7hx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:25:57Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj namespace: kserve-ci-e2e-test resourceVersion: "15027" uid: c348e295-37a9-4e1d-a806-e255f953dbbe kind: Event lastTimestamp: "2026-04-28T19:25:57Z" message: Add eth0 [10.134.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:25:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:25:57Z" name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj.18aa9bd894b11654 namespace: kserve-ci-e2e-test resourceVersion: "15034" uid: 54c003cd-ead1-4bd1-94d9-1bbad19fe77f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:25:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: c348e295-37a9-4e1d-a806-e255f953dbbe kind: Event lastTimestamp: "2026-04-28T19:25:57Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3460f014e7dc0a9d3daafe0716ca9eadf865f2892e0a5103d0b876da9f34891f" already present on machine metadata: creationTimestamp: "2026-04-28T19:25:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:25:57Z" name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj.18aa9bd8962e374b namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: 744188b7-26a7-4cd4-b41d-1d58911e1daa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:25:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: c348e295-37a9-4e1d-a806-e255f953dbbe kind: Event lastTimestamp: "2026-04-28T19:25:57Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-28T19:25:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:25:57Z" name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj.18aa9bd8996952a3 namespace: kserve-ci-e2e-test resourceVersion: "15041" uid: d59d5579-6621-452a-9a43-4e3a47d30c97 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:25:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: c348e295-37a9-4e1d-a806-e255f953dbbe kind: Event lastTimestamp: "2026-04-28T19:25:57Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-28T19:25:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:25:57Z" name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj.18aa9bd899f086f0 namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: 6e43f3b5-4371-4fc6-a604-0684b7b69219 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:26:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: c348e295-37a9-4e1d-a806-e255f953dbbe kind: Event lastTimestamp: "2026-04-28T19:26:01Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-28T19:26:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:26:01Z" name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj.18aa9bd992986baf namespace: kserve-ci-e2e-test resourceVersion: "15117" uid: a8edbc03-4de2-447d-b2d9-8591fc0827cd reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:26:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: c348e295-37a9-4e1d-a806-e255f953dbbe kind: Event lastTimestamp: "2026-04-28T19:26:20Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 19.235s (19.235s including waiting). Image size: 1306329851 bytes.' metadata: creationTimestamp: "2026-04-28T19:26:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:26:20Z" name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj.18aa9bde0d247a4d namespace: kserve-ci-e2e-test resourceVersion: "15426" uid: 0e978f9d-ea9b-4cd3-bddb-cc535b712176 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:26:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: c348e295-37a9-4e1d-a806-e255f953dbbe kind: Event lastTimestamp: "2026-04-28T19:26:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:26:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:26:21Z" name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj.18aa9bde1005f1e6 namespace: kserve-ci-e2e-test resourceVersion: "15427" uid: 7802cf67-53ce-4569-8674-0a0cd5bc6c43 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:26:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: c348e295-37a9-4e1d-a806-e255f953dbbe kind: Event lastTimestamp: "2026-04-28T19:26:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:26:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:26:21Z" name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj.18aa9bde107ad389 namespace: kserve-ci-e2e-test resourceVersion: "15428" uid: 89c00dfa-726b-4d85-bd18-740000ad65bf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:26:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: c348e295-37a9-4e1d-a806-e255f953dbbe kind: Event lastTimestamp: "2026-04-28T19:26:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:26:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:26:21Z" name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj.18aa9bde10874552 namespace: kserve-ci-e2e-test resourceVersion: "15429" uid: 56de8e3b-7b08-4f13-8d1b-1d24ee92651d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:26:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: c348e295-37a9-4e1d-a806-e255f953dbbe kind: Event lastTimestamp: "2026-04-28T19:26:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:26:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:26:21Z" name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj.18aa9bde136a6a51 namespace: kserve-ci-e2e-test resourceVersion: "15431" uid: bfe2e889-1eb8-4ef0-b2a2-0a2b7f9a1f90 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:26:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: c348e295-37a9-4e1d-a806-e255f953dbbe kind: Event lastTimestamp: "2026-04-28T19:26:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:26:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:26:21Z" name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj.18aa9bde13f16ce6 namespace: kserve-ci-e2e-test resourceVersion: "15432" uid: b0f94d0c-22f8-46fd-a45a-6e5bb0988dae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-28T19:26:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: c348e295-37a9-4e1d-a806-e255f953dbbe kind: Event lastTimestamp: "2026-04-28T19:27:48Z" message: 'Readiness probe failed: dial tcp 10.134.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:26:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:48Z" name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj.18aa9bde7a92cce8 namespace: kserve-ci-e2e-test resourceVersion: "16847" uid: 0a8af3b2-1c94-4d33-a45d-7b40411e60f9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: c348e295-37a9-4e1d-a806-e255f953dbbe kind: Event lastTimestamp: "2026-04-28T19:27:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:46Z" name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj.18aa9bf1e408014f namespace: kserve-ci-e2e-test resourceVersion: "16717" uid: ad46e5da-ee5d-4534-b779-3b40f8f2c12a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: c348e295-37a9-4e1d-a806-e255f953dbbe kind: Event lastTimestamp: "2026-04-28T19:27:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:46Z" name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj.18aa9bf1e4089356 namespace: kserve-ci-e2e-test resourceVersion: "16719" uid: 20a0e228-45aa-456f-b7b0-7784373071b4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: c348e295-37a9-4e1d-a806-e255f953dbbe kind: Event lastTimestamp: "2026-04-28T19:27:48Z" message: 'Readiness probe failed: Get "https://10.134.0.19:8643/healthz": dial tcp 10.134.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:27:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:48Z" name: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj.18aa9bf280bfc398 namespace: kserve-ci-e2e-test resourceVersion: "16846" uid: a96b0e37-f015-4ffa-84c4-3b87fc364ca7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:25:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-357ae-predictor-f454f465 namespace: kserve-ci-e2e-test resourceVersion: "14999" uid: 14acfe6e-adf5-4da4-996c-cc3a4c6b54b6 kind: Event lastTimestamp: "2026-04-28T19:25:57Z" message: 'Created pod: isvc-xgboost-graph-raw-357ae-predictor-f454f465-2dszj' metadata: creationTimestamp: "2026-04-28T19:25:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:25:57Z" name: isvc-xgboost-graph-raw-357ae-predictor-f454f465.18aa9bd87ab95b20 namespace: kserve-ci-e2e-test resourceVersion: "15019" uid: 4f110da9-8d67-4da7-b489-5447e43a55c7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:25:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-357ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "14995" uid: 5db405ba-d8fa-4311-947c-98361bbb936f kind: Event lastTimestamp: "2026-04-28T19:25:56Z" message: Scaled up replica set isvc-xgboost-graph-raw-357ae-predictor-f454f465 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:25:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:25:56Z" name: isvc-xgboost-graph-raw-357ae-predictor.18aa9bd8646077e7 namespace: kserve-ci-e2e-test resourceVersion: "15001" uid: 7ec1ab89-1120-4ad3-9bc9-96be64c7f460 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:26:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-357ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: 308fcb39-c56f-4eae-ae9b-bc7256929e7e kind: Event lastTimestamp: "2026-04-28T19:26:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:26:41Z" name: isvc-xgboost-graph-raw-357ae-predictor.18aa9bdbe3072362 namespace: kserve-ci-e2e-test resourceVersion: "15733" uid: d4eebbaf-e581-4975-8ab5-3d7a750c19c5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:26:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-357ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: 308fcb39-c56f-4eae-ae9b-bc7256929e7e kind: Event lastTimestamp: "2026-04-28T19:26:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:26:41Z" name: isvc-xgboost-graph-raw-357ae-predictor.18aa9bdbe379de8b namespace: kserve-ci-e2e-test resourceVersion: "15734" uid: e0efbf9b-8c8d-4639-a473-d249587e6355 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-28T19:26:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-357ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: 308fcb39-c56f-4eae-ae9b-bc7256929e7e kind: Event lastTimestamp: "2026-04-28T19:27:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:27:41Z" name: isvc-xgboost-graph-raw-357ae-predictor.18aa9be65fdc214f namespace: kserve-ci-e2e-test resourceVersion: "16613" uid: c2d0af2b-08fc-4b93-963f-1b6f8ab2aca9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-28T19:26:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-357ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: 308fcb39-c56f-4eae-ae9b-bc7256929e7e kind: Event lastTimestamp: "2026-04-28T19:27:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:27:41Z" name: isvc-xgboost-graph-raw-357ae-predictor.18aa9be66044a687 namespace: kserve-ci-e2e-test resourceVersion: "16614" uid: 7c18ff7c-b320-4370-a8c4-e754aebf4c27 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 25 eventTime: null firstTimestamp: "2026-04-28T19:25:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-357ae namespace: kserve-ci-e2e-test resourceVersion: "14975" uid: c66b5cc6-7b0f-4692-87ae-25edf60fa37f kind: Event lastTimestamp: "2026-04-28T19:26:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:25:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:26:13Z" name: isvc-xgboost-graph-raw-357ae.18aa9bd864383f7a namespace: kserve-ci-e2e-test resourceVersion: "15330" uid: 71f16d8b-d3f7-44a4-bfa1-fdbe8922dfb4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-357ae namespace: kserve-ci-e2e-test resourceVersion: "16413" uid: c66b5cc6-7b0f-4692-87ae-25edf60fa37f kind: Event lastTimestamp: "2026-04-28T19:27:29Z" message: InferenceService [isvc-xgboost-graph-raw-357ae] is Ready metadata: creationTimestamp: "2026-04-28T19:27:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:27:29Z" name: isvc-xgboost-graph-raw-357ae.18aa9bedf5bc42f6 namespace: kserve-ci-e2e-test resourceVersion: "16416" uid: 23be59f2-51d0-42bf-98b7-cac1c46dacda reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:27:47.469308Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t namespace: kserve-ci-e2e-test resourceVersion: "16800" uid: 9c16c9c9-c169-4cd6-afea-4cd267985347 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t to ip-10-0-134-36.ec2.internal metadata: creationTimestamp: "2026-04-28T19:27:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:27:47Z" name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t.18aa9bf231b36a1e namespace: kserve-ci-e2e-test resourceVersion: "16804" uid: 3e573bf7-d783-487b-bc90-f0ef0850f3da reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f8cb4f8bc-qf7hx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:47Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t namespace: kserve-ci-e2e-test resourceVersion: "16808" uid: 9c16c9c9-c169-4cd6-afea-4cd267985347 kind: Event lastTimestamp: "2026-04-28T19:27:47Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:27:47Z" name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t.18aa9bf24afbb90d namespace: kserve-ci-e2e-test resourceVersion: "16816" uid: 8c52a55d-c1b4-4b22-9863-57146c2bd790 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t namespace: kserve-ci-e2e-test resourceVersion: "16802" uid: 9c16c9c9-c169-4cd6-afea-4cd267985347 kind: Event lastTimestamp: "2026-04-28T19:27:47Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3460f014e7dc0a9d3daafe0716ca9eadf865f2892e0a5103d0b876da9f34891f" already present on machine metadata: creationTimestamp: "2026-04-28T19:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:47Z" name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t.18aa9bf24c5bc78e namespace: kserve-ci-e2e-test resourceVersion: "16818" uid: 278b20cd-271e-4c87-9af8-18fe2a005c06 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t namespace: kserve-ci-e2e-test resourceVersion: "16802" uid: 9c16c9c9-c169-4cd6-afea-4cd267985347 kind: Event lastTimestamp: "2026-04-28T19:27:47Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-28T19:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:47Z" name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t.18aa9bf24f96ddc6 namespace: kserve-ci-e2e-test resourceVersion: "16819" uid: 4463589e-c46d-4bd5-a692-6e89637c4301 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t namespace: kserve-ci-e2e-test resourceVersion: "16802" uid: 9c16c9c9-c169-4cd6-afea-4cd267985347 kind: Event lastTimestamp: "2026-04-28T19:27:47Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-28T19:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:47Z" name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t.18aa9bf2500b26c6 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: ea447387-7f73-4ecd-bf34-13be6922f5ac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t namespace: kserve-ci-e2e-test resourceVersion: "16802" uid: 9c16c9c9-c169-4cd6-afea-4cd267985347 kind: Event lastTimestamp: "2026-04-28T19:27:52Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-04-28T19:27:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:52Z" name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t.18aa9bf343b7245a namespace: kserve-ci-e2e-test resourceVersion: "16912" uid: 72a1c95c-1e2c-43fc-b04f-e6e35590d2af reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t namespace: kserve-ci-e2e-test resourceVersion: "16802" uid: 9c16c9c9-c169-4cd6-afea-4cd267985347 kind: Event lastTimestamp: "2026-04-28T19:27:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:27:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:52Z" name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t.18aa9bf3483d680f namespace: kserve-ci-e2e-test resourceVersion: "16916" uid: eeb38b13-1c60-49a9-8bbb-868865bd2113 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t namespace: kserve-ci-e2e-test resourceVersion: "16802" uid: 9c16c9c9-c169-4cd6-afea-4cd267985347 kind: Event lastTimestamp: "2026-04-28T19:27:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:27:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:52Z" name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t.18aa9bf348edfd0d namespace: kserve-ci-e2e-test resourceVersion: "16918" uid: 861f3236-74cc-4653-b8f8-76befaaf0dc7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t namespace: kserve-ci-e2e-test resourceVersion: "16802" uid: 9c16c9c9-c169-4cd6-afea-4cd267985347 kind: Event lastTimestamp: "2026-04-28T19:27:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:27:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:52Z" name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t.18aa9bf348fcf881 namespace: kserve-ci-e2e-test resourceVersion: "16919" uid: f3fe2357-6a40-4489-a35d-2350f34f594d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t namespace: kserve-ci-e2e-test resourceVersion: "16802" uid: 9c16c9c9-c169-4cd6-afea-4cd267985347 kind: Event lastTimestamp: "2026-04-28T19:27:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:27:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:52Z" name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t.18aa9bf34c36afdd namespace: kserve-ci-e2e-test resourceVersion: "16922" uid: 97d0a15e-0c54-4c31-a86d-24365bca10b4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t namespace: kserve-ci-e2e-test resourceVersion: "16802" uid: 9c16c9c9-c169-4cd6-afea-4cd267985347 kind: Event lastTimestamp: "2026-04-28T19:27:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:27:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:52Z" name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t.18aa9bf34caabaa1 namespace: kserve-ci-e2e-test resourceVersion: "16923" uid: 1d4cd062-16ad-4785-9ed1-1af1aa967562 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-28T19:27:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t namespace: kserve-ci-e2e-test resourceVersion: "16802" uid: 9c16c9c9-c169-4cd6-afea-4cd267985347 kind: Event lastTimestamp: "2026-04-28T19:29:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:27:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:29Z" name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t.18aa9bf37fdc8629 namespace: kserve-ci-e2e-test resourceVersion: "18443" uid: b305b877-ba2f-45e1-a75b-001299f05bec reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t namespace: kserve-ci-e2e-test resourceVersion: "16802" uid: 9c16c9c9-c169-4cd6-afea-4cd267985347 kind: Event lastTimestamp: "2026-04-28T19:29:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:26Z" name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t.18aa9c09414dcbf2 namespace: kserve-ci-e2e-test resourceVersion: "18350" uid: bcb76080-0d46-49db-b05b-93691e202f43 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t namespace: kserve-ci-e2e-test resourceVersion: "16802" uid: 9c16c9c9-c169-4cd6-afea-4cd267985347 kind: Event lastTimestamp: "2026-04-28T19:29:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:26Z" name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t.18aa9c09414e3a60 namespace: kserve-ci-e2e-test resourceVersion: "18352" uid: 75521de7-0874-48ff-97b1-b344341af319 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t namespace: kserve-ci-e2e-test resourceVersion: "16802" uid: 9c16c9c9-c169-4cd6-afea-4cd267985347 kind: Event lastTimestamp: "2026-04-28T19:29:29Z" message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": dial tcp 10.134.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:29:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:29Z" name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t.18aa9c09da2ec84b namespace: kserve-ci-e2e-test resourceVersion: "18442" uid: 362a6315-715f-4017-aa8b-4ad688c095ff reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69 namespace: kserve-ci-e2e-test resourceVersion: "16765" uid: f478f620-089f-4e2d-85b2-d3793bb4d912 kind: Event lastTimestamp: "2026-04-28T19:27:47Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69-66x2t' metadata: creationTimestamp: "2026-04-28T19:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:27:47Z" name: isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69.18aa9bf23146817c namespace: kserve-ci-e2e-test resourceVersion: "16801" uid: f225e0cf-898e-41b5-83a6-7c95ba4e6019 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-5628a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16763" uid: 8faa72e0-8d1d-426a-a6c5-47b782daf056 kind: Event lastTimestamp: "2026-04-28T19:27:47Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-5628a-predictor-6fcd96cc69 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:27:47Z" name: isvc-xgboost-graph-raw-hpa-5628a-predictor.18aa9bf21b198ef6 namespace: kserve-ci-e2e-test resourceVersion: "16768" uid: bccaf227-82bc-4014-bb5d-49bf66474faa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:28:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-5628a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16766" uid: 41476528-af86-4a8d-aa19-f8e4ed11c3ce kind: Event lastTimestamp: "2026-04-28T19:28:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:28:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:28:17Z" name: isvc-xgboost-graph-raw-hpa-5628a-predictor.18aa9bf599d2dd35 namespace: kserve-ci-e2e-test resourceVersion: "17315" uid: fb7e34b1-ef8c-4dfb-9d9a-442601fbd051 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:28:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-5628a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16766" uid: 41476528-af86-4a8d-aa19-f8e4ed11c3ce kind: Event lastTimestamp: "2026-04-28T19:28:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:28:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:28:17Z" name: isvc-xgboost-graph-raw-hpa-5628a-predictor.18aa9bf59a3c106a namespace: kserve-ci-e2e-test resourceVersion: "17316" uid: ee289c9b-7699-4e75-b627-a6292f11f443 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-28T19:28:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-5628a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17093" uid: 41476528-af86-4a8d-aa19-f8e4ed11c3ce kind: Event lastTimestamp: "2026-04-28T19:29:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:28:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:29:17Z" name: isvc-xgboost-graph-raw-hpa-5628a-predictor.18aa9bfc97ad8e2b namespace: kserve-ci-e2e-test resourceVersion: "18186" uid: 0c18ff0b-c68a-4823-b736-263d85abb504 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-28T19:28:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-5628a-predictor namespace: kserve-ci-e2e-test resourceVersion: "17093" uid: 41476528-af86-4a8d-aa19-f8e4ed11c3ce kind: Event lastTimestamp: "2026-04-28T19:29:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:28:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:29:17Z" name: isvc-xgboost-graph-raw-hpa-5628a-predictor.18aa9bfc98135dbf namespace: kserve-ci-e2e-test resourceVersion: "18187" uid: 9e4357aa-c113-43fb-8f6f-d34c735913c7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 25 eventTime: null firstTimestamp: "2026-04-28T19:27:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-5628a namespace: kserve-ci-e2e-test resourceVersion: "16741" uid: 8604e468-b316-4d21-bad4-4f331c6951a4 kind: Event lastTimestamp: "2026-04-28T19:28:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:28:04Z" name: isvc-xgboost-graph-raw-hpa-5628a.18aa9bf21b338a18 namespace: kserve-ci-e2e-test resourceVersion: "17125" uid: dd7d5822-b239-4aa0-a7c8-337fa4235d52 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:28:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-5628a namespace: kserve-ci-e2e-test resourceVersion: "17907" uid: 8604e468-b316-4d21-bad4-4f331c6951a4 kind: Event lastTimestamp: "2026-04-28T19:28:59Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-5628a] is Ready metadata: creationTimestamp: "2026-04-28T19:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:28:59Z" name: isvc-xgboost-graph-raw-hpa-5628a.18aa9c02f0577716 namespace: kserve-ci-e2e-test resourceVersion: "17908" uid: edb2ae1e-46e8-4540-8207-a3be379774c1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:29:26.990289Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-a8bff-predictor-6d6466ff77-7bxkq namespace: kserve-ci-e2e-test resourceVersion: "18390" uid: c323af27-9a6e-4bba-86e5-6e631c80460b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-a8bff-predictor-6d6466ff77-7bxkq to ip-10-0-134-36.ec2.internal metadata: creationTimestamp: "2026-04-28T19:29:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:29:26Z" name: message-dumper-raw-a8bff-predictor-6d6466ff77-7bxkq.18aa9c095d9d1b6a namespace: kserve-ci-e2e-test resourceVersion: "18394" uid: 2e10a87f-2d3f-4b9d-b51b-b9c1f2bb8e93 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f8cb4f8bc-qf7hx source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:29:27Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-a8bff-predictor-6d6466ff77-7bxkq namespace: kserve-ci-e2e-test resourceVersion: "18392" uid: c323af27-9a6e-4bba-86e5-6e631c80460b kind: Event lastTimestamp: "2026-04-28T19:29:27Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "message-dumper-raw-a8bff-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-28T19:29:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:27Z" name: message-dumper-raw-a8bff-predictor-6d6466ff77-7bxkq.18aa9c09687df8ae namespace: kserve-ci-e2e-test resourceVersion: "18412" uid: ae628f13-f7af-4811-b93f-16b62a343a0a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:28Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-a8bff-predictor-6d6466ff77-7bxkq namespace: kserve-ci-e2e-test resourceVersion: "18398" uid: c323af27-9a6e-4bba-86e5-6e631c80460b kind: Event lastTimestamp: "2026-04-28T19:29:28Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:29:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:29:28Z" name: message-dumper-raw-a8bff-predictor-6d6466ff77-7bxkq.18aa9c09d02f997c namespace: kserve-ci-e2e-test resourceVersion: "18435" uid: 0342b73f-2b73-4f1e-a242-dc65b716d7dd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-a8bff-predictor-6d6466ff77-7bxkq namespace: kserve-ci-e2e-test resourceVersion: "18392" uid: c323af27-9a6e-4bba-86e5-6e631c80460b kind: Event lastTimestamp: "2026-04-28T19:29:28Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-04-28T19:29:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:28Z" name: message-dumper-raw-a8bff-predictor-6d6466ff77-7bxkq.18aa9c09d1765749 namespace: kserve-ci-e2e-test resourceVersion: "18437" uid: 30f35522-fe96-48c5-afcb-6d01ec4fc85e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-a8bff-predictor-6d6466ff77-7bxkq namespace: kserve-ci-e2e-test resourceVersion: "18392" uid: c323af27-9a6e-4bba-86e5-6e631c80460b kind: Event lastTimestamp: "2026-04-28T19:29:29Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.058s (1.058s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-04-28T19:29:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:29Z" name: message-dumper-raw-a8bff-predictor-6d6466ff77-7bxkq.18aa9c0a108d8e58 namespace: kserve-ci-e2e-test resourceVersion: "18454" uid: ac1e2739-952c-4169-9b3d-cd8fce661cea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-a8bff-predictor-6d6466ff77-7bxkq namespace: kserve-ci-e2e-test resourceVersion: "18392" uid: c323af27-9a6e-4bba-86e5-6e631c80460b kind: Event lastTimestamp: "2026-04-28T19:29:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:30Z" name: message-dumper-raw-a8bff-predictor-6d6466ff77-7bxkq.18aa9c0a14635ee4 namespace: kserve-ci-e2e-test resourceVersion: "18455" uid: 0afe1db2-7671-4f99-93c3-045914d8c3fc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-a8bff-predictor-6d6466ff77-7bxkq namespace: kserve-ci-e2e-test resourceVersion: "18392" uid: c323af27-9a6e-4bba-86e5-6e631c80460b kind: Event lastTimestamp: "2026-04-28T19:29:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:30Z" name: message-dumper-raw-a8bff-predictor-6d6466ff77-7bxkq.18aa9c0a14d8c56f namespace: kserve-ci-e2e-test resourceVersion: "18457" uid: 103f9cc3-c0b4-4f84-9b4d-07fb8d27dcfc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-a8bff-predictor-6d6466ff77-7bxkq namespace: kserve-ci-e2e-test resourceVersion: "18392" uid: c323af27-9a6e-4bba-86e5-6e631c80460b kind: Event lastTimestamp: "2026-04-28T19:29:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:30Z" name: message-dumper-raw-a8bff-predictor-6d6466ff77-7bxkq.18aa9c0a14e65006 namespace: kserve-ci-e2e-test resourceVersion: "18458" uid: d4ac39bb-bae6-4098-b6ff-05d13a65f77d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-a8bff-predictor-6d6466ff77-7bxkq namespace: kserve-ci-e2e-test resourceVersion: "18392" uid: c323af27-9a6e-4bba-86e5-6e631c80460b kind: Event lastTimestamp: "2026-04-28T19:29:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:30Z" name: message-dumper-raw-a8bff-predictor-6d6466ff77-7bxkq.18aa9c0a1aea2cf0 namespace: kserve-ci-e2e-test resourceVersion: "18462" uid: 93cfeb22-9a0c-45cc-9d1a-e42bfea2a302 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-a8bff-predictor-6d6466ff77-7bxkq namespace: kserve-ci-e2e-test resourceVersion: "18392" uid: c323af27-9a6e-4bba-86e5-6e631c80460b kind: Event lastTimestamp: "2026-04-28T19:29:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:30Z" name: message-dumper-raw-a8bff-predictor-6d6466ff77-7bxkq.18aa9c0a1b67cc8a namespace: kserve-ci-e2e-test resourceVersion: "18463" uid: 438f4a13-bf32-45f1-a7f7-f6340c8ad43f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-a8bff-predictor-6d6466ff77-7bxkq namespace: kserve-ci-e2e-test resourceVersion: "18392" uid: c323af27-9a6e-4bba-86e5-6e631c80460b kind: Event lastTimestamp: "2026-04-28T19:31:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:11Z" name: message-dumper-raw-a8bff-predictor-6d6466ff77-7bxkq.18aa9c21bc49e5bd namespace: kserve-ci-e2e-test resourceVersion: "19993" uid: 85961c37-bb51-4901-8bab-68a590562a82 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:31:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-a8bff-predictor-6d6466ff77-7bxkq namespace: kserve-ci-e2e-test resourceVersion: "18392" uid: c323af27-9a6e-4bba-86e5-6e631c80460b kind: Event lastTimestamp: "2026-04-28T19:31:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:31:11Z" name: message-dumper-raw-a8bff-predictor-6d6466ff77-7bxkq.18aa9c21bc4a529c namespace: kserve-ci-e2e-test resourceVersion: "19994" uid: 4b0dc7e1-17bf-4c08-873f-3781b5e05e95 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-a8bff-predictor-6d6466ff77 namespace: kserve-ci-e2e-test resourceVersion: "18363" uid: 90f8d6ec-15d4-42d5-a011-c558a705fce6 kind: Event lastTimestamp: "2026-04-28T19:29:26Z" message: 'Created pod: message-dumper-raw-a8bff-predictor-6d6466ff77-7bxkq' metadata: creationTimestamp: "2026-04-28T19:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:29:26Z" name: message-dumper-raw-a8bff-predictor-6d6466ff77.18aa9c095d376081 namespace: kserve-ci-e2e-test resourceVersion: "18391" uid: 6f23047e-a8aa-4ae9-bf2e-e125de0bd309 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-a8bff-predictor namespace: kserve-ci-e2e-test resourceVersion: "18361" uid: 5bf59062-ff76-43d1-ab1c-6920700fcee9 kind: Event lastTimestamp: "2026-04-28T19:29:26Z" message: Scaled up replica set message-dumper-raw-a8bff-predictor-6d6466ff77 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:29:26Z" name: message-dumper-raw-a8bff-predictor.18aa9c0954c46870 namespace: kserve-ci-e2e-test resourceVersion: "18365" uid: 3e39670a-9669-47c3-8af2-e3bde407d5e1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:29:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-a8bff-predictor namespace: kserve-ci-e2e-test resourceVersion: "18362" uid: 2d243375-d28f-46ca-a1aa-c71052431878 kind: Event lastTimestamp: "2026-04-28T19:29:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:29:56Z" name: message-dumper-raw-a8bff-predictor.18aa9c0cd39a247a namespace: kserve-ci-e2e-test resourceVersion: "18923" uid: acc489c4-4042-4e45-9f13-1ac7884651a2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:29:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-a8bff-predictor namespace: kserve-ci-e2e-test resourceVersion: "18362" uid: 2d243375-d28f-46ca-a1aa-c71052431878 kind: Event lastTimestamp: "2026-04-28T19:29:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:29:56Z" name: message-dumper-raw-a8bff-predictor.18aa9c0cd3fb8e41 namespace: kserve-ci-e2e-test resourceVersion: "18924" uid: 795033dd-83ba-45cc-9aa6-2e678996bfaf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 25 eventTime: null firstTimestamp: "2026-04-28T19:29:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-a8bff namespace: kserve-ci-e2e-test resourceVersion: "18351" uid: 6fb41062-49fc-4687-8cbf-2f4c06a2ff3b kind: Event lastTimestamp: "2026-04-28T19:29:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:29:35Z" name: message-dumper-raw-a8bff.18aa9c0954a06304 namespace: kserve-ci-e2e-test resourceVersion: "18558" uid: bf4af464-1dcd-419b-bf29-6b6a0c9e80aa reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-a8bff namespace: kserve-ci-e2e-test resourceVersion: "18608" uid: 6fb41062-49fc-4687-8cbf-2f4c06a2ff3b kind: Event lastTimestamp: "2026-04-28T19:29:38Z" message: InferenceService [message-dumper-raw-a8bff] is Ready metadata: creationTimestamp: "2026-04-28T19:29:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:29:38Z" name: message-dumper-raw-a8bff.18aa9c0c1653f369 namespace: kserve-ci-e2e-test resourceVersion: "18610" uid: b6e096c1-8dd6-4181-959a-d6388d7424b7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:27:36.212088Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-357ae-5d6b99669d-w894m namespace: kserve-ci-e2e-test resourceVersion: "16515" uid: 25e44e91-07da-4b0d-90a4-a4cb65b8d799 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-357ae-5d6b99669d-w894m to ip-10-0-132-160.ec2.internal metadata: creationTimestamp: "2026-04-28T19:27:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:27:36Z" name: model-chainer-raw-357ae-5d6b99669d-w894m.18aa9bef92b7dd4a namespace: kserve-ci-e2e-test resourceVersion: "16520" uid: 5cd9c8fb-d693-4629-9113-d21e964998ea reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f8cb4f8bc-qf7hx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:36Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-357ae-5d6b99669d-w894m namespace: kserve-ci-e2e-test resourceVersion: "16517" uid: 25e44e91-07da-4b0d-90a4-a4cb65b8d799 kind: Event lastTimestamp: "2026-04-28T19:27:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-357ae-serving-cert" not found' metadata: creationTimestamp: "2026-04-28T19:27:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:36Z" name: model-chainer-raw-357ae-5d6b99669d-w894m.18aa9bef9ef3fa0d namespace: kserve-ci-e2e-test resourceVersion: "16526" uid: e1c65cdd-0330-4114-873e-0bebfa56f86b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-160.ec2.internal source: component: kubelet host: ip-10-0-132-160.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:37Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-357ae-5d6b99669d-w894m namespace: kserve-ci-e2e-test resourceVersion: "16524" uid: 25e44e91-07da-4b0d-90a4-a4cb65b8d799 kind: Event lastTimestamp: "2026-04-28T19:27:37Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:27:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:27:37Z" name: model-chainer-raw-357ae-5d6b99669d-w894m.18aa9befd0014d88 namespace: kserve-ci-e2e-test resourceVersion: "16541" uid: ac49b992-8bdf-4de4-a8b9-a67c24b955a9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-357ae} kind: Pod name: model-chainer-raw-357ae-5d6b99669d-w894m namespace: kserve-ci-e2e-test resourceVersion: "16517" uid: 25e44e91-07da-4b0d-90a4-a4cb65b8d799 kind: Event lastTimestamp: "2026-04-28T19:27:37Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:6a7f2e0065ff588673ad687410eb1f2b5ccfca9b42b71ed586d2fac5a64a6ca4" metadata: creationTimestamp: "2026-04-28T19:27:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:37Z" name: model-chainer-raw-357ae-5d6b99669d-w894m.18aa9befd2a0c4eb namespace: kserve-ci-e2e-test resourceVersion: "16544" uid: cf4a3b43-19f6-475e-92a7-a8744fedb21a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-160.ec2.internal source: component: kubelet host: ip-10-0-132-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-357ae} kind: Pod name: model-chainer-raw-357ae-5d6b99669d-w894m namespace: kserve-ci-e2e-test resourceVersion: "16517" uid: 25e44e91-07da-4b0d-90a4-a4cb65b8d799 kind: Event lastTimestamp: "2026-04-28T19:27:39Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:6a7f2e0065ff588673ad687410eb1f2b5ccfca9b42b71ed586d2fac5a64a6ca4" in 2.528s (2.528s including waiting). Image size: 216346748 bytes.' metadata: creationTimestamp: "2026-04-28T19:27:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:39Z" name: model-chainer-raw-357ae-5d6b99669d-w894m.18aa9bf069574c30 namespace: kserve-ci-e2e-test resourceVersion: "16577" uid: 0b73147e-e4b6-4937-8c07-adc0b3c17996 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-160.ec2.internal source: component: kubelet host: ip-10-0-132-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-357ae} kind: Pod name: model-chainer-raw-357ae-5d6b99669d-w894m namespace: kserve-ci-e2e-test resourceVersion: "16517" uid: 25e44e91-07da-4b0d-90a4-a4cb65b8d799 kind: Event lastTimestamp: "2026-04-28T19:27:39Z" message: 'Created container: model-chainer-raw-357ae' metadata: creationTimestamp: "2026-04-28T19:27:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:39Z" name: model-chainer-raw-357ae-5d6b99669d-w894m.18aa9bf06c71817d namespace: kserve-ci-e2e-test resourceVersion: "16578" uid: 4b9ef8fe-14e5-43e2-ad4f-5851e50aeb25 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-160.ec2.internal source: component: kubelet host: ip-10-0-132-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-357ae} kind: Pod name: model-chainer-raw-357ae-5d6b99669d-w894m namespace: kserve-ci-e2e-test resourceVersion: "16517" uid: 25e44e91-07da-4b0d-90a4-a4cb65b8d799 kind: Event lastTimestamp: "2026-04-28T19:27:39Z" message: Started container model-chainer-raw-357ae metadata: creationTimestamp: "2026-04-28T19:27:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:39Z" name: model-chainer-raw-357ae-5d6b99669d-w894m.18aa9bf06cecf5fe namespace: kserve-ci-e2e-test resourceVersion: "16579" uid: 2a0fbb7f-a41c-4b39-93dd-bd36fc38aeea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-160.ec2.internal source: component: kubelet host: ip-10-0-132-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-357ae} kind: Pod name: model-chainer-raw-357ae-5d6b99669d-w894m namespace: kserve-ci-e2e-test resourceVersion: "16517" uid: 25e44e91-07da-4b0d-90a4-a4cb65b8d799 kind: Event lastTimestamp: "2026-04-28T19:27:46Z" message: Stopping container model-chainer-raw-357ae metadata: creationTimestamp: "2026-04-28T19:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:27:46Z" name: model-chainer-raw-357ae-5d6b99669d-w894m.18aa9bf1e1b76072 namespace: kserve-ci-e2e-test resourceVersion: "16702" uid: 94883995-3e86-4afc-9486-848ac8502d72 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-160.ec2.internal source: component: kubelet host: ip-10-0-132-160.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:27:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-357ae} kind: Pod name: model-chainer-raw-357ae-5d6b99669d-w894m namespace: kserve-ci-e2e-test resourceVersion: "16517" uid: 25e44e91-07da-4b0d-90a4-a4cb65b8d799 kind: Event lastTimestamp: "2026-04-28T19:28:16Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-28T19:27:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:28:16Z" name: model-chainer-raw-357ae-5d6b99669d-w894m.18aa9bf3063b9646 namespace: kserve-ci-e2e-test resourceVersion: "17293" uid: a7331890-ca57-408d-b07e-d0fdef5bf279 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-160.ec2.internal source: component: kubelet host: ip-10-0-132-160.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-357ae-5d6b99669d namespace: kserve-ci-e2e-test resourceVersion: "16512" uid: 0915651b-ee3a-485d-9503-ac2edada8ca2 kind: Event lastTimestamp: "2026-04-28T19:27:36Z" message: 'Created pod: model-chainer-raw-357ae-5d6b99669d-w894m' metadata: creationTimestamp: "2026-04-28T19:27:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:27:36Z" name: model-chainer-raw-357ae-5d6b99669d.18aa9bef9256439d namespace: kserve-ci-e2e-test resourceVersion: "16518" uid: 17f41e39-cd61-4962-90c1-081f18956a3b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-357ae namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: b2f2f0d8-f09a-44e8-9327-c9b79d260bf4 kind: Event lastTimestamp: "2026-04-28T19:27:36Z" message: Scaled up replica set model-chainer-raw-357ae-5d6b99669d from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:27:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:27:36Z" name: model-chainer-raw-357ae.18aa9bef917179d5 namespace: kserve-ci-e2e-test resourceVersion: "16513" uid: c0e8b0ea-fa8c-4af9-a545-2cc305ef70f7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:27:36Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-357ae namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: 39cd1f48-1aa4-433d-b437-5c734e4a24f3 kind: Event lastTimestamp: "2026-04-28T19:27:36Z" message: InferenceGraph [model-chainer-raw-357ae] is Ready metadata: creationTimestamp: "2026-04-28T19:27:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:27:36Z" name: model-chainer-raw-357ae.18aa9befaf2c5f72 namespace: kserve-ci-e2e-test resourceVersion: "16537" uid: 5b0f2481-19a9-43ee-8dc6-fcec1f6e53d3 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:29:16.469149Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-5628a-687c9b99db-wsj4p namespace: kserve-ci-e2e-test resourceVersion: "18156" uid: 24891f2e-3b1e-4852-a49d-2450a35264ba kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-5628a-687c9b99db-wsj4p to ip-10-0-132-160.ec2.internal metadata: creationTimestamp: "2026-04-28T19:29:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:29:16Z" name: model-chainer-raw-hpa-5628a-687c9b99db-wsj4p.18aa9c06ea81495e namespace: kserve-ci-e2e-test resourceVersion: "18161" uid: de344750-0f94-4a9e-ac6c-4da56557e0b8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f8cb4f8bc-qf7hx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:16Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-5628a-687c9b99db-wsj4p namespace: kserve-ci-e2e-test resourceVersion: "18166" uid: 24891f2e-3b1e-4852-a49d-2450a35264ba kind: Event lastTimestamp: "2026-04-28T19:29:16Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:29:16Z" name: model-chainer-raw-hpa-5628a-687c9b99db-wsj4p.18aa9c0703e505d9 namespace: kserve-ci-e2e-test resourceVersion: "18176" uid: 738cdae0-2ab4-4e9c-8c5a-7af41f35c07d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-5628a} kind: Pod name: model-chainer-raw-hpa-5628a-687c9b99db-wsj4p namespace: kserve-ci-e2e-test resourceVersion: "18158" uid: 24891f2e-3b1e-4852-a49d-2450a35264ba kind: Event lastTimestamp: "2026-04-28T19:29:16Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:6a7f2e0065ff588673ad687410eb1f2b5ccfca9b42b71ed586d2fac5a64a6ca4" already present on machine metadata: creationTimestamp: "2026-04-28T19:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:16Z" name: model-chainer-raw-hpa-5628a-687c9b99db-wsj4p.18aa9c07058fccef namespace: kserve-ci-e2e-test resourceVersion: "18178" uid: 6cb9267f-031e-4958-9cd3-b85bcb488601 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-160.ec2.internal source: component: kubelet host: ip-10-0-132-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-5628a} kind: Pod name: model-chainer-raw-hpa-5628a-687c9b99db-wsj4p namespace: kserve-ci-e2e-test resourceVersion: "18158" uid: 24891f2e-3b1e-4852-a49d-2450a35264ba kind: Event lastTimestamp: "2026-04-28T19:29:16Z" message: 'Created container: model-chainer-raw-hpa-5628a' metadata: creationTimestamp: "2026-04-28T19:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:16Z" name: model-chainer-raw-hpa-5628a-687c9b99db-wsj4p.18aa9c0708b70e56 namespace: kserve-ci-e2e-test resourceVersion: "18182" uid: e23d9827-4a69-4bf3-8fc3-65d571b1df4a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-160.ec2.internal source: component: kubelet host: ip-10-0-132-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-5628a} kind: Pod name: model-chainer-raw-hpa-5628a-687c9b99db-wsj4p namespace: kserve-ci-e2e-test resourceVersion: "18158" uid: 24891f2e-3b1e-4852-a49d-2450a35264ba kind: Event lastTimestamp: "2026-04-28T19:29:16Z" message: Started container model-chainer-raw-hpa-5628a metadata: creationTimestamp: "2026-04-28T19:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:16Z" name: model-chainer-raw-hpa-5628a-687c9b99db-wsj4p.18aa9c070930205c namespace: kserve-ci-e2e-test resourceVersion: "18183" uid: b2481fc7-938d-44ba-80a8-2471656b14b1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-160.ec2.internal source: component: kubelet host: ip-10-0-132-160.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-5628a} kind: Pod name: model-chainer-raw-hpa-5628a-687c9b99db-wsj4p namespace: kserve-ci-e2e-test resourceVersion: "18158" uid: 24891f2e-3b1e-4852-a49d-2450a35264ba kind: Event lastTimestamp: "2026-04-28T19:29:26Z" message: Stopping container model-chainer-raw-hpa-5628a metadata: creationTimestamp: "2026-04-28T19:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:26Z" name: model-chainer-raw-hpa-5628a-687c9b99db-wsj4p.18aa9c09375688d3 namespace: kserve-ci-e2e-test resourceVersion: "18331" uid: 42f0ff88-dcb5-48eb-add2-7720345280b0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-160.ec2.internal source: component: kubelet host: ip-10-0-132-160.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-28T19:29:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-5628a} kind: Pod name: model-chainer-raw-hpa-5628a-687c9b99db-wsj4p namespace: kserve-ci-e2e-test resourceVersion: "18158" uid: 24891f2e-3b1e-4852-a49d-2450a35264ba kind: Event lastTimestamp: "2026-04-28T19:29:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-28T19:29:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:29:53Z" name: model-chainer-raw-hpa-5628a-687c9b99db-wsj4p.18aa9c09af5695c7 namespace: kserve-ci-e2e-test resourceVersion: "18870" uid: 8ddec723-feb5-431f-8fac-3346277a89a3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-160.ec2.internal source: component: kubelet host: ip-10-0-132-160.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-5628a-687c9b99db namespace: kserve-ci-e2e-test resourceVersion: "18153" uid: efc305f4-04a1-47eb-8969-e9ab92582a2a kind: Event lastTimestamp: "2026-04-28T19:29:16Z" message: 'Created pod: model-chainer-raw-hpa-5628a-687c9b99db-wsj4p' metadata: creationTimestamp: "2026-04-28T19:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:29:16Z" name: model-chainer-raw-hpa-5628a-687c9b99db.18aa9c06ea0ae4f9 namespace: kserve-ci-e2e-test resourceVersion: "18157" uid: ad1a3121-7abb-4690-9c1e-30fb94ee142c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-5628a namespace: kserve-ci-e2e-test resourceVersion: "18151" uid: 93fab8c6-1958-4e14-95a2-55094b7bec33 kind: Event lastTimestamp: "2026-04-28T19:29:16Z" message: Scaled up replica set model-chainer-raw-hpa-5628a-687c9b99db from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:29:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:29:16Z" name: model-chainer-raw-hpa-5628a.18aa9c06e92edf54 namespace: kserve-ci-e2e-test resourceVersion: "18155" uid: 54e410aa-ecd6-4826-ba5e-b43771a65909 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:16Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-5628a namespace: kserve-ci-e2e-test resourceVersion: "18180" uid: 13634ab0-84de-4dc1-a1ef-f9eeeeba4641 kind: Event lastTimestamp: "2026-04-28T19:29:16Z" message: InferenceGraph [model-chainer-raw-hpa-5628a] is Ready metadata: creationTimestamp: "2026-04-28T19:29:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:29:17Z" name: model-chainer-raw-hpa-5628a.18aa9c0706eeffc9 namespace: kserve-ci-e2e-test resourceVersion: "18185" uid: aebd8b77-222f-49ff-98d0-4461bbe832cd reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:17Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-5628a namespace: kserve-ci-e2e-test resourceVersion: "18141" uid: 13634ab0-84de-4dc1-a1ef-f9eeeeba4641 kind: Event lastTimestamp: "2026-04-28T19:29:17Z" message: 'Failed to update status for InferenceGraph "model-chainer-raw-hpa-5628a": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer-raw-hpa-5628a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:29:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:29:17Z" name: model-chainer-raw-hpa-5628a.18aa9c072676f634 namespace: kserve-ci-e2e-test resourceVersion: "18195" uid: a50f17a4-0aa6-4de9-a2ac-9eaef88c3b4d reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:29:17Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-5628a namespace: kserve-ci-e2e-test resourceVersion: "18141" uid: 13634ab0-84de-4dc1-a1ef-f9eeeeba4641 kind: Event lastTimestamp: "2026-04-28T19:29:17Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer-raw-hpa-5628a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:29:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:29:17Z" name: model-chainer-raw-hpa-5628a.18aa9c07267744f4 namespace: kserve-ci-e2e-test resourceVersion: "18197" uid: accf74c1-7015-47da-b9c1-6d93a74c2793 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:35:18.251146Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt namespace: kserve-ci-e2e-test resourceVersion: "23082" uid: a799eefe-a5f4-45cb-af20-50c943b0b9e4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-3e269-predictor-75dc5c954c-zt7nt to ip-10-0-134-36.ec2.internal metadata: creationTimestamp: "2026-04-28T19:35:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:35:18Z" name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt.18aa9c5b26646c94 namespace: kserve-ci-e2e-test resourceVersion: "23088" uid: e31be824-1990-4a9a-aaec-9894ae04b821 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f8cb4f8bc-qf7hx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:18Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt namespace: kserve-ci-e2e-test resourceVersion: "23085" uid: a799eefe-a5f4-45cb-af20-50c943b0b9e4 kind: Event lastTimestamp: "2026-04-28T19:35:18Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-3e269-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-28T19:35:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:35:18Z" name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt.18aa9c5b32d5e0e1 namespace: kserve-ci-e2e-test resourceVersion: "23104" uid: da0fe4e1-5085-441e-b94d-9c20936b812e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:19Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt namespace: kserve-ci-e2e-test resourceVersion: "23092" uid: a799eefe-a5f4-45cb-af20-50c943b0b9e4 kind: Event lastTimestamp: "2026-04-28T19:35:19Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:35:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:35:19Z" name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt.18aa9c5b630f3871 namespace: kserve-ci-e2e-test resourceVersion: "23121" uid: 843b604a-178b-45c4-a309-2d13546e4761 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt namespace: kserve-ci-e2e-test resourceVersion: "23085" uid: a799eefe-a5f4-45cb-af20-50c943b0b9e4 kind: Event lastTimestamp: "2026-04-28T19:35:19Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3460f014e7dc0a9d3daafe0716ca9eadf865f2892e0a5103d0b876da9f34891f" already present on machine metadata: creationTimestamp: "2026-04-28T19:35:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:35:19Z" name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt.18aa9c5b6473b5fa namespace: kserve-ci-e2e-test resourceVersion: "23124" uid: 5ab4a7b3-e7cb-425d-bd1d-89f6bf60a99b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt namespace: kserve-ci-e2e-test resourceVersion: "23085" uid: a799eefe-a5f4-45cb-af20-50c943b0b9e4 kind: Event lastTimestamp: "2026-04-28T19:35:19Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-28T19:35:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:35:19Z" name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt.18aa9c5b670fb714 namespace: kserve-ci-e2e-test resourceVersion: "23127" uid: baafc7d2-35bf-4a4b-b0f2-44ed2beda488 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt namespace: kserve-ci-e2e-test resourceVersion: "23085" uid: a799eefe-a5f4-45cb-af20-50c943b0b9e4 kind: Event lastTimestamp: "2026-04-28T19:35:19Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-28T19:35:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:35:19Z" name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt.18aa9c5b6782863b namespace: kserve-ci-e2e-test resourceVersion: "23128" uid: bcac34fd-ef3e-4129-9b64-a42d605afdd0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt namespace: kserve-ci-e2e-test resourceVersion: "23085" uid: a799eefe-a5f4-45cb-af20-50c943b0b9e4 kind: Event lastTimestamp: "2026-04-28T19:35:23Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-28T19:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:35:23Z" name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt.18aa9c5c57cb2be0 namespace: kserve-ci-e2e-test resourceVersion: "23195" uid: a66e9d41-f747-4095-b672-4bb2d149bb61 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt namespace: kserve-ci-e2e-test resourceVersion: "23085" uid: a799eefe-a5f4-45cb-af20-50c943b0b9e4 kind: Event lastTimestamp: "2026-04-28T19:35:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:35:23Z" name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt.18aa9c5c5ac0af2e namespace: kserve-ci-e2e-test resourceVersion: "23197" uid: 4c41adab-a99d-4987-a750-23fc14d9f6b7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt namespace: kserve-ci-e2e-test resourceVersion: "23085" uid: a799eefe-a5f4-45cb-af20-50c943b0b9e4 kind: Event lastTimestamp: "2026-04-28T19:35:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:35:23Z" name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt.18aa9c5c5b336f4a namespace: kserve-ci-e2e-test resourceVersion: "23198" uid: 0645df47-e402-4dc9-a68b-35b5646333be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt namespace: kserve-ci-e2e-test resourceVersion: "23085" uid: a799eefe-a5f4-45cb-af20-50c943b0b9e4 kind: Event lastTimestamp: "2026-04-28T19:35:23Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:35:23Z" name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt.18aa9c5c5b3efa75 namespace: kserve-ci-e2e-test resourceVersion: "23199" uid: feb9fbca-7205-4e5d-870f-d78cb4fcb202 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt namespace: kserve-ci-e2e-test resourceVersion: "23085" uid: a799eefe-a5f4-45cb-af20-50c943b0b9e4 kind: Event lastTimestamp: "2026-04-28T19:35:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:35:23Z" name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt.18aa9c5c5e025e74 namespace: kserve-ci-e2e-test resourceVersion: "23200" uid: a3b307c9-bf26-4157-8ddb-067da597c407 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt namespace: kserve-ci-e2e-test resourceVersion: "23085" uid: a799eefe-a5f4-45cb-af20-50c943b0b9e4 kind: Event lastTimestamp: "2026-04-28T19:35:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:35:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:35:23Z" name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt.18aa9c5c5e7538a3 namespace: kserve-ci-e2e-test resourceVersion: "23201" uid: 3c654bf4-330a-42f1-b87b-8df2573753ed reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-28T19:35:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt namespace: kserve-ci-e2e-test resourceVersion: "23085" uid: a799eefe-a5f4-45cb-af20-50c943b0b9e4 kind: Event lastTimestamp: "2026-04-28T19:36:40Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:35:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:36:40Z" name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt.18aa9c5c93b7e5cf namespace: kserve-ci-e2e-test resourceVersion: "24325" uid: 802660ee-82b2-4a12-81c1-8a700c2c73dc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:36:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt namespace: kserve-ci-e2e-test resourceVersion: "23085" uid: a799eefe-a5f4-45cb-af20-50c943b0b9e4 kind: Event lastTimestamp: "2026-04-28T19:36:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:36:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:36:39Z" name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt.18aa9c6df8065b36 namespace: kserve-ci-e2e-test resourceVersion: "24290" uid: fc209fee-b756-468c-906d-4e93ce91a8b1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:36:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt namespace: kserve-ci-e2e-test resourceVersion: "23085" uid: a799eefe-a5f4-45cb-af20-50c943b0b9e4 kind: Event lastTimestamp: "2026-04-28T19:36:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:36:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:36:39Z" name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt.18aa9c6df806ce00 namespace: kserve-ci-e2e-test resourceVersion: "24291" uid: 82e62243-4fd5-452b-81a5-2494cfd909ee reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:36:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt namespace: kserve-ci-e2e-test resourceVersion: "23085" uid: a799eefe-a5f4-45cb-af20-50c943b0b9e4 kind: Event lastTimestamp: "2026-04-28T19:36:40Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:36:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:36:40Z" name: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt.18aa9c6e45c7c47e namespace: kserve-ci-e2e-test resourceVersion: "24324" uid: 380daa8f-a5dd-4f71-bd3a-84a7c4599582 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-3e269-predictor-75dc5c954c namespace: kserve-ci-e2e-test resourceVersion: "23074" uid: cc69c773-928b-46ec-a093-2009ed6e43e0 kind: Event lastTimestamp: "2026-04-28T19:35:18Z" message: 'Created pod: raw-sklearn-3e269-predictor-75dc5c954c-zt7nt' metadata: creationTimestamp: "2026-04-28T19:35:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:35:18Z" name: raw-sklearn-3e269-predictor-75dc5c954c.18aa9c5b25de0a1b namespace: kserve-ci-e2e-test resourceVersion: "23084" uid: 2d35b167-8674-4578-91ae-02bd0738c605 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-3e269-predictor namespace: kserve-ci-e2e-test resourceVersion: "23072" uid: e0ab9ca0-9505-4c41-975e-5bdbba12c908 kind: Event lastTimestamp: "2026-04-28T19:35:18Z" message: Scaled up replica set raw-sklearn-3e269-predictor-75dc5c954c from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:35:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:35:18Z" name: raw-sklearn-3e269-predictor.18aa9c5b23868843 namespace: kserve-ci-e2e-test resourceVersion: "23076" uid: 563ec1a7-b9c9-4e00-9caa-de2a4143c943 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-3e269-predictor namespace: kserve-ci-e2e-test resourceVersion: "23073" uid: c3dc4708-6261-4d35-acac-4c008c070759 kind: Event lastTimestamp: "2026-04-28T19:35:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:35:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:35:33Z" name: raw-sklearn-3e269-predictor.18aa9c5ea2929ce1 namespace: kserve-ci-e2e-test resourceVersion: "23347" uid: 415cfdfe-dfa5-43c0-a84f-619c37a6e473 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-3e269-predictor namespace: kserve-ci-e2e-test resourceVersion: "23073" uid: c3dc4708-6261-4d35-acac-4c008c070759 kind: Event lastTimestamp: "2026-04-28T19:35:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:35:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:35:33Z" name: raw-sklearn-3e269-predictor.18aa9c5ea2f3c88d namespace: kserve-ci-e2e-test resourceVersion: "23348" uid: 09309de5-903d-4273-985b-994b5ea3f179 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-28T19:35:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-3e269-predictor namespace: kserve-ci-e2e-test resourceVersion: "23346" uid: c3dc4708-6261-4d35-acac-4c008c070759 kind: Event lastTimestamp: "2026-04-28T19:36:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:35:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:36:33Z" name: raw-sklearn-3e269-predictor.18aa9c6221c42b96 namespace: kserve-ci-e2e-test resourceVersion: "24174" uid: d7fcf851-3272-42b0-8148-20da5bdde11a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-28T19:35:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-3e269-predictor namespace: kserve-ci-e2e-test resourceVersion: "23346" uid: c3dc4708-6261-4d35-acac-4c008c070759 kind: Event lastTimestamp: "2026-04-28T19:36:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:35:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:36:33Z" name: raw-sklearn-3e269-predictor.18aa9c62222515de namespace: kserve-ci-e2e-test resourceVersion: "24175" uid: 3af4fe01-c7c7-46ee-ae72-76b888f6829a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 22 eventTime: null firstTimestamp: "2026-04-28T19:35:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-3e269 namespace: kserve-ci-e2e-test resourceVersion: "23062" uid: 77275667-d306-4174-8593-54d8484a64db kind: Event lastTimestamp: "2026-04-28T19:35:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:35:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:35:24Z" name: raw-sklearn-3e269.18aa9c5b233de16c namespace: kserve-ci-e2e-test resourceVersion: "23228" uid: 0d12d7e3-6444-4f08-b7e1-6e1560b4a1f7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:35:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-3e269 namespace: kserve-ci-e2e-test resourceVersion: "23062" uid: 77275667-d306-4174-8593-54d8484a64db kind: Event lastTimestamp: "2026-04-28T19:35:18Z" message: 'Failed to update status for InferenceService "raw-sklearn-3e269": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-3e269": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:35:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:35:18Z" name: raw-sklearn-3e269.18aa9c5b24b77c8b namespace: kserve-ci-e2e-test resourceVersion: "23100" uid: 6233269e-a97a-4bea-a556-1338475b50c3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:35:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-3e269 namespace: kserve-ci-e2e-test resourceVersion: "23083" uid: 77275667-d306-4174-8593-54d8484a64db kind: Event lastTimestamp: "2026-04-28T19:35:18Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-3e269": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-28T19:35:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:35:18Z" name: raw-sklearn-3e269.18aa9c5b2e9e9f72 namespace: kserve-ci-e2e-test resourceVersion: "23102" uid: 4d673f9f-a199-41ee-ac3f-a04e85941076 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:36:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-3e269 namespace: kserve-ci-e2e-test resourceVersion: "24138" uid: 77275667-d306-4174-8593-54d8484a64db kind: Event lastTimestamp: "2026-04-28T19:36:30Z" message: InferenceService [raw-sklearn-3e269] is Ready metadata: creationTimestamp: "2026-04-28T19:36:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:36:30Z" name: raw-sklearn-3e269.18aa9c6c046e0c63 namespace: kserve-ci-e2e-test resourceVersion: "24140" uid: f7b017d2-34ad-493c-bed6-b8b8ba8c553e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-28T19:36:39.286251Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf namespace: kserve-ci-e2e-test resourceVersion: "24293" uid: 1bbc6869-5623-45de-a9f8-e60c722b36a7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf to ip-10-0-134-36.ec2.internal metadata: creationTimestamp: "2026-04-28T19:36:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-28T19:36:39Z" name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf.18aa9c6e0475fba6 namespace: kserve-ci-e2e-test resourceVersion: "24298" uid: 998a2bee-2e90-47d3-849e-9f2c23c2b59d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-f8cb4f8bc-qf7hx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:36:39Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf namespace: kserve-ci-e2e-test resourceVersion: "24302" uid: 1bbc6869-5623-45de-a9f8-e60c722b36a7 kind: Event lastTimestamp: "2026-04-28T19:36:39Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-28T19:36:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-28T19:36:39Z" name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf.18aa9c6e1ef978e2 namespace: kserve-ci-e2e-test resourceVersion: "24308" uid: b668405b-2185-41ab-ac57-5ea658b4a886 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:36:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf namespace: kserve-ci-e2e-test resourceVersion: "24295" uid: 1bbc6869-5623-45de-a9f8-e60c722b36a7 kind: Event lastTimestamp: "2026-04-28T19:36:39Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3460f014e7dc0a9d3daafe0716ca9eadf865f2892e0a5103d0b876da9f34891f" already present on machine metadata: creationTimestamp: "2026-04-28T19:36:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:36:39Z" name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf.18aa9c6e20993466 namespace: kserve-ci-e2e-test resourceVersion: "24310" uid: 29d12525-f6ab-4931-ae05-f6e8e1bcbb16 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:36:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf namespace: kserve-ci-e2e-test resourceVersion: "24295" uid: 1bbc6869-5623-45de-a9f8-e60c722b36a7 kind: Event lastTimestamp: "2026-04-28T19:36:39Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-28T19:36:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:36:39Z" name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf.18aa9c6e23cc7cd4 namespace: kserve-ci-e2e-test resourceVersion: "24311" uid: 499605ba-84e1-4ffc-a622-41c0d6f0f789 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:36:39Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf namespace: kserve-ci-e2e-test resourceVersion: "24295" uid: 1bbc6869-5623-45de-a9f8-e60c722b36a7 kind: Event lastTimestamp: "2026-04-28T19:36:39Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-28T19:36:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:36:39Z" name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf.18aa9c6e243cda9e namespace: kserve-ci-e2e-test resourceVersion: "24312" uid: 67e1f55c-e40a-4c38-974b-e1343759a457 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:36:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf namespace: kserve-ci-e2e-test resourceVersion: "24295" uid: 1bbc6869-5623-45de-a9f8-e60c722b36a7 kind: Event lastTimestamp: "2026-04-28T19:36:44Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1404" already present on machine metadata: creationTimestamp: "2026-04-28T19:36:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:36:44Z" name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf.18aa9c6f41973768 namespace: kserve-ci-e2e-test resourceVersion: "24397" uid: 2eb740ac-bfdc-4007-bb17-59e050187529 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:36:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf namespace: kserve-ci-e2e-test resourceVersion: "24295" uid: 1bbc6869-5623-45de-a9f8-e60c722b36a7 kind: Event lastTimestamp: "2026-04-28T19:36:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-28T19:36:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:36:44Z" name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf.18aa9c6f4495f01a namespace: kserve-ci-e2e-test resourceVersion: "24399" uid: 3680f9b8-3231-4010-9bd3-1b404dab1bef reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:36:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf namespace: kserve-ci-e2e-test resourceVersion: "24295" uid: 1bbc6869-5623-45de-a9f8-e60c722b36a7 kind: Event lastTimestamp: "2026-04-28T19:36:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-28T19:36:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:36:44Z" name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf.18aa9c6f450a1986 namespace: kserve-ci-e2e-test resourceVersion: "24400" uid: ebdd56e0-5330-43a9-8887-280d13df9e43 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:36:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf namespace: kserve-ci-e2e-test resourceVersion: "24295" uid: 1bbc6869-5623-45de-a9f8-e60c722b36a7 kind: Event lastTimestamp: "2026-04-28T19:36:44Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-28T19:36:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:36:44Z" name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf.18aa9c6f45179f76 namespace: kserve-ci-e2e-test resourceVersion: "24401" uid: c5822496-30de-4b4f-b214-75edfa8fd3cd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:36:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf namespace: kserve-ci-e2e-test resourceVersion: "24295" uid: 1bbc6869-5623-45de-a9f8-e60c722b36a7 kind: Event lastTimestamp: "2026-04-28T19:36:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-28T19:36:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:36:44Z" name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf.18aa9c6f47d55f22 namespace: kserve-ci-e2e-test resourceVersion: "24403" uid: f28846cc-9870-445a-a447-4be812cb604f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:36:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf namespace: kserve-ci-e2e-test resourceVersion: "24295" uid: 1bbc6869-5623-45de-a9f8-e60c722b36a7 kind: Event lastTimestamp: "2026-04-28T19:36:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:36:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:36:44Z" name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf.18aa9c6f4846a8cb namespace: kserve-ci-e2e-test resourceVersion: "24404" uid: 156adbb2-0bc9-4c37-a5de-762fde572aa7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-28T19:36:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf namespace: kserve-ci-e2e-test resourceVersion: "24295" uid: 1bbc6869-5623-45de-a9f8-e60c722b36a7 kind: Event lastTimestamp: "2026-04-28T19:38:01Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:36:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:38:01Z" name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf.18aa9c6f7d7fb280 namespace: kserve-ci-e2e-test resourceVersion: "25500" uid: d421aee8-51b3-4179-a0d7-ca9bbcd21e70 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:37:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf namespace: kserve-ci-e2e-test resourceVersion: "24295" uid: 1bbc6869-5623-45de-a9f8-e60c722b36a7 kind: Event lastTimestamp: "2026-04-28T19:37:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-28T19:37:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:37:58Z" name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf.18aa9c807a2cee48 namespace: kserve-ci-e2e-test resourceVersion: "25446" uid: 810393b8-b787-4ce8-b22d-c96a7e28cfd8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:37:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf namespace: kserve-ci-e2e-test resourceVersion: "24295" uid: 1bbc6869-5623-45de-a9f8-e60c722b36a7 kind: Event lastTimestamp: "2026-04-28T19:37:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-28T19:37:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:37:58Z" name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf.18aa9c807a2d5b4e namespace: kserve-ci-e2e-test resourceVersion: "25447" uid: 018b9d0f-f5e5-4b4a-8cbc-e28d9b869333 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:38:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf namespace: kserve-ci-e2e-test resourceVersion: "24295" uid: 1bbc6869-5623-45de-a9f8-e60c722b36a7 kind: Event lastTimestamp: "2026-04-28T19:38:01Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-28T19:38:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-28T19:38:01Z" name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf.18aa9c812f98f8d4 namespace: kserve-ci-e2e-test resourceVersion: "25499" uid: 2a31ada0-b0e8-46a9-add2-c4fe52baf4e5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-36.ec2.internal source: component: kubelet host: ip-10-0-134-36.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:36:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5 namespace: kserve-ci-e2e-test resourceVersion: "24263" uid: 09e93519-4c89-4d87-929b-8ac602564f6b kind: Event lastTimestamp: "2026-04-28T19:36:39Z" message: 'Created pod: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5-gj7gf' metadata: creationTimestamp: "2026-04-28T19:36:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:36:39Z" name: raw-sklearn-runtime-cdca2-predictor-65dc66dfd5.18aa9c6e03f4adb2 namespace: kserve-ci-e2e-test resourceVersion: "24294" uid: 24920a50-9d97-4cca-a15d-8c071be16d5d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:36:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-cdca2-predictor namespace: kserve-ci-e2e-test resourceVersion: "24260" uid: 9c439e61-e950-49a7-a5b4-782a9cb478bd kind: Event lastTimestamp: "2026-04-28T19:36:38Z" message: Scaled up replica set raw-sklearn-runtime-cdca2-predictor-65dc66dfd5 from 0 to 1 metadata: creationTimestamp: "2026-04-28T19:36:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:36:38Z" name: raw-sklearn-runtime-cdca2-predictor.18aa9c6ded6523f2 namespace: kserve-ci-e2e-test resourceVersion: "24264" uid: 8ac313a7-6da6-41b5-b4e2-f1e390bd0cff reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:36:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-cdca2-predictor namespace: kserve-ci-e2e-test resourceVersion: "24262" uid: 5c8e1897-fe2e-4163-9cc8-88c114cb4f67 kind: Event lastTimestamp: "2026-04-28T19:37:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:36:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:37:23Z" name: raw-sklearn-runtime-cdca2-predictor.18aa9c716bddce26 namespace: kserve-ci-e2e-test resourceVersion: "24953" uid: 919f3c27-97a6-43c5-aeda-ac951774aee9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-28T19:36:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-cdca2-predictor namespace: kserve-ci-e2e-test resourceVersion: "24262" uid: 5c8e1897-fe2e-4163-9cc8-88c114cb4f67 kind: Event lastTimestamp: "2026-04-28T19:37:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-28T19:36:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:37:23Z" name: raw-sklearn-runtime-cdca2-predictor.18aa9c716c40208e namespace: kserve-ci-e2e-test resourceVersion: "24954" uid: e6ea1b1d-a431-4ccd-bcf3-90b8d188356c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:37:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-cdca2-predictor namespace: kserve-ci-e2e-test resourceVersion: "24542" uid: 5c8e1897-fe2e-4163-9cc8-88c114cb4f67 kind: Event lastTimestamp: "2026-04-28T19:37:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:37:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:37:53Z" name: raw-sklearn-runtime-cdca2-predictor.18aa9c7be9652d50 namespace: kserve-ci-e2e-test resourceVersion: "25370" uid: 95c66cf2-2895-491f-bd8e-0494883958d1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-28T19:37:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-cdca2-predictor namespace: kserve-ci-e2e-test resourceVersion: "24542" uid: 5c8e1897-fe2e-4163-9cc8-88c114cb4f67 kind: Event lastTimestamp: "2026-04-28T19:37:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-28T19:37:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-28T19:37:53Z" name: raw-sklearn-runtime-cdca2-predictor.18aa9c7be9c5b85c namespace: kserve-ci-e2e-test resourceVersion: "25371" uid: 46dadf85-7f72-435d-8bb8-2351a835f3e2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 25 eventTime: null firstTimestamp: "2026-04-28T19:36:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-cdca2 namespace: kserve-ci-e2e-test resourceVersion: "24251" uid: f87bfd6f-78ae-445c-8c37-3f137a6d5dfe kind: Event lastTimestamp: "2026-04-28T19:36:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-28T19:36:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:36:47Z" name: raw-sklearn-runtime-cdca2.18aa9c6ded39b7f9 namespace: kserve-ci-e2e-test resourceVersion: "24454" uid: 172803a5-4cde-4b79-a4ad-0d7951f9975c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-28T19:37:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-cdca2 namespace: kserve-ci-e2e-test resourceVersion: "25338" uid: f87bfd6f-78ae-445c-8c37-3f137a6d5dfe kind: Event lastTimestamp: "2026-04-28T19:37:51Z" message: InferenceService [raw-sklearn-runtime-cdca2] is Ready metadata: creationTimestamp: "2026-04-28T19:37:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-28T19:37:51Z" name: raw-sklearn-runtime-cdca2.18aa9c7ee645b28b namespace: kserve-ci-e2e-test resourceVersion: "25339" uid: dd29c2d8-e035-4736-b6fb-4d582a680ab5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "25646"