--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:41:28.226886Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-75b0d-699b59b6f-snfrm namespace: kserve-ci-e2e-test resourceVersion: "21412" uid: ee921bcc-b5c8-44b2-b035-839e57c39143 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-75b0d-699b59b6f-snfrm to ip-10-0-128-21.ec2.internal metadata: creationTimestamp: "2026-04-24T21:41:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:41:28Z" name: ensemble-graph-75b0d-699b59b6f-snfrm.18a968eb66df6ad4 namespace: kserve-ci-e2e-test resourceVersion: "21418" uid: 63e881c0-76bd-4685-ab31-59f114711cc2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:28Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-75b0d-699b59b6f-snfrm namespace: kserve-ci-e2e-test resourceVersion: "21421" uid: ee921bcc-b5c8-44b2-b035-839e57c39143 kind: Event lastTimestamp: "2026-04-24T21:41:28Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:41:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:41:28Z" name: ensemble-graph-75b0d-699b59b6f-snfrm.18a968eb8032ce64 namespace: kserve-ci-e2e-test resourceVersion: "21432" uid: 445573e9-a85c-4ce7-8f27-378de13307f4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-75b0d} kind: Pod name: ensemble-graph-75b0d-699b59b6f-snfrm namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: ee921bcc-b5c8-44b2-b035-839e57c39143 kind: Event lastTimestamp: "2026-04-24T21:41:28Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f2021b9f79dbd151723ae4c386dd28bedd0fff23f6daa0825bdc58598fc71887" already present on machine metadata: creationTimestamp: "2026-04-24T21:41:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:28Z" name: ensemble-graph-75b0d-699b59b6f-snfrm.18a968eb81706a69 namespace: kserve-ci-e2e-test resourceVersion: "21434" uid: 13413f41-446d-488e-b18c-5fb60eea054f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-75b0d} kind: Pod name: ensemble-graph-75b0d-699b59b6f-snfrm namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: ee921bcc-b5c8-44b2-b035-839e57c39143 kind: Event lastTimestamp: "2026-04-24T21:41:28Z" message: 'Created container: ensemble-graph-75b0d' metadata: creationTimestamp: "2026-04-24T21:41:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:28Z" name: ensemble-graph-75b0d-699b59b6f-snfrm.18a968eb856ab04b namespace: kserve-ci-e2e-test resourceVersion: "21435" uid: c0d3c617-7c6b-4009-9b3d-2e28b8082bfc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-75b0d} kind: Pod name: ensemble-graph-75b0d-699b59b6f-snfrm namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: ee921bcc-b5c8-44b2-b035-839e57c39143 kind: Event lastTimestamp: "2026-04-24T21:41:28Z" message: Started container ensemble-graph-75b0d metadata: creationTimestamp: "2026-04-24T21:41:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:28Z" name: ensemble-graph-75b0d-699b59b6f-snfrm.18a968eb85df9a1c namespace: kserve-ci-e2e-test resourceVersion: "21436" uid: 62c49c09-1241-4eb6-a680-35ab03026a81 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-75b0d} kind: Pod name: ensemble-graph-75b0d-699b59b6f-snfrm namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: ee921bcc-b5c8-44b2-b035-839e57c39143 kind: Event lastTimestamp: "2026-04-24T21:49:42Z" message: Stopping container ensemble-graph-75b0d metadata: creationTimestamp: "2026-04-24T21:49:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:42Z" name: ensemble-graph-75b0d-699b59b6f-snfrm.18a9695e9483413b namespace: kserve-ci-e2e-test resourceVersion: "24912" uid: 85814dd4-3958-47fd-a329-d7b62e5fbe4a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:49:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-75b0d} kind: Pod name: ensemble-graph-75b0d-699b59b6f-snfrm namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: ee921bcc-b5c8-44b2-b035-839e57c39143 kind: Event lastTimestamp: "2026-04-24T21:50:10Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:10Z" name: ensemble-graph-75b0d-699b59b6f-snfrm.18a9695f12a83cfd namespace: kserve-ci-e2e-test resourceVersion: "25531" uid: 285f6245-6113-44aa-b362-2cf5d1efe666 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-75b0d-699b59b6f namespace: kserve-ci-e2e-test resourceVersion: "21408" uid: d083956a-0ad9-4745-82af-a302d3cfa53d kind: Event lastTimestamp: "2026-04-24T21:41:28Z" message: 'Created pod: ensemble-graph-75b0d-699b59b6f-snfrm' metadata: creationTimestamp: "2026-04-24T21:41:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:28Z" name: ensemble-graph-75b0d-699b59b6f.18a968eb6670d1a0 namespace: kserve-ci-e2e-test resourceVersion: "21413" uid: 8e9b1bea-e940-4781-8c42-a08469e83366 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-75b0d namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: 79e29705-ab73-4251-aff7-e8aac3815389 kind: Event lastTimestamp: "2026-04-24T21:41:28Z" message: Scaled up replica set ensemble-graph-75b0d-699b59b6f from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:41:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:28Z" name: ensemble-graph-75b0d.18a968eb65bb652b namespace: kserve-ci-e2e-test resourceVersion: "21410" uid: c930ca27-477a-477e-9b04-f74f8847ce1e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:28Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-75b0d namespace: kserve-ci-e2e-test resourceVersion: "21423" uid: 89e08882-6643-48eb-bc24-f1d913ec12cf kind: Event lastTimestamp: "2026-04-24T21:41:28Z" message: InferenceGraph [ensemble-graph-75b0d] is Ready metadata: creationTimestamp: "2026-04-24T21:41:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:41:28Z" name: ensemble-graph-75b0d.18a968eb6ac57ff8 namespace: kserve-ci-e2e-test resourceVersion: "21425" uid: 621b4850-c2db-4e77-9437-899c454ed8ab reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:41:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-75b0d namespace: kserve-ci-e2e-test resourceVersion: "21407" uid: 9ed548da-d9d0-4a02-865a-c4e3deb450f0 kind: Event lastTimestamp: "2026-04-24T21:42:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:42:13Z" name: ensemble-graph-75b0d.18a968eee437ea1d namespace: kserve-ci-e2e-test resourceVersion: "21842" uid: c94f4415-6877-4a43-a71f-a5a5f4c57dfd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:41:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-75b0d namespace: kserve-ci-e2e-test resourceVersion: "21407" uid: 9ed548da-d9d0-4a02-865a-c4e3deb450f0 kind: Event lastTimestamp: "2026-04-24T21:42:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:42:13Z" name: ensemble-graph-75b0d.18a968eee4a0e561 namespace: kserve-ci-e2e-test resourceVersion: "21843" uid: 26a27ade-4530-430b-9289-39c8e0c87f32 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:40:17.986289Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-f3231-7b5968d9d6-rbjc2 namespace: kserve-ci-e2e-test resourceVersion: "20444" uid: bf598abc-362a-4194-9c7c-8f53960c1ec8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-f3231-7b5968d9d6-rbjc2 to ip-10-0-128-21.ec2.internal metadata: creationTimestamp: "2026-04-24T21:40:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:40:17Z" name: ensemble-graph-f3231-7b5968d9d6-rbjc2.18a968db0c350322 namespace: kserve-ci-e2e-test resourceVersion: "20449" uid: 4aad227e-2771-4ed2-988a-c2a232de3f9b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:18Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-f3231-7b5968d9d6-rbjc2 namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: bf598abc-362a-4194-9c7c-8f53960c1ec8 kind: Event lastTimestamp: "2026-04-24T21:40:18Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-f3231-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:18Z" name: ensemble-graph-f3231-7b5968d9d6-rbjc2.18a968db14b5a2ef namespace: kserve-ci-e2e-test resourceVersion: "20465" uid: c51ca4fe-07c8-43ae-9afc-3ee707adc147 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:19Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-f3231-7b5968d9d6-rbjc2 namespace: kserve-ci-e2e-test resourceVersion: "20452" uid: bf598abc-362a-4194-9c7c-8f53960c1ec8 kind: Event lastTimestamp: "2026-04-24T21:40:19Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:40:19Z" name: ensemble-graph-f3231-7b5968d9d6-rbjc2.18a968db490e5566 namespace: kserve-ci-e2e-test resourceVersion: "20472" uid: 7148bb27-b56d-4a8d-9c7f-f8cee50a6837 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-f3231} kind: Pod name: ensemble-graph-f3231-7b5968d9d6-rbjc2 namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: bf598abc-362a-4194-9c7c-8f53960c1ec8 kind: Event lastTimestamp: "2026-04-24T21:40:19Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f2021b9f79dbd151723ae4c386dd28bedd0fff23f6daa0825bdc58598fc71887" already present on machine metadata: creationTimestamp: "2026-04-24T21:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:19Z" name: ensemble-graph-f3231-7b5968d9d6-rbjc2.18a968db4a5604cf namespace: kserve-ci-e2e-test resourceVersion: "20474" uid: 8257b775-86cc-4766-8f4c-6c1078741f6c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-f3231} kind: Pod name: ensemble-graph-f3231-7b5968d9d6-rbjc2 namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: bf598abc-362a-4194-9c7c-8f53960c1ec8 kind: Event lastTimestamp: "2026-04-24T21:40:19Z" message: 'Created container: ensemble-graph-f3231' metadata: creationTimestamp: "2026-04-24T21:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:19Z" name: ensemble-graph-f3231-7b5968d9d6-rbjc2.18a968db4db80dde namespace: kserve-ci-e2e-test resourceVersion: "20478" uid: a679c27d-7f2c-4d86-b8b9-a687350ba3b5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-f3231} kind: Pod name: ensemble-graph-f3231-7b5968d9d6-rbjc2 namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: bf598abc-362a-4194-9c7c-8f53960c1ec8 kind: Event lastTimestamp: "2026-04-24T21:40:19Z" message: Started container ensemble-graph-f3231 metadata: creationTimestamp: "2026-04-24T21:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:19Z" name: ensemble-graph-f3231-7b5968d9d6-rbjc2.18a968db4e90bd9f namespace: kserve-ci-e2e-test resourceVersion: "20479" uid: 9ff977d0-5037-4b9b-8095-8e9b5383b973 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-f3231} kind: Pod name: ensemble-graph-f3231-7b5968d9d6-rbjc2 namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: bf598abc-362a-4194-9c7c-8f53960c1ec8 kind: Event lastTimestamp: "2026-04-24T21:40:28Z" message: Stopping container ensemble-graph-f3231 metadata: creationTimestamp: "2026-04-24T21:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:28Z" name: ensemble-graph-f3231-7b5968d9d6-rbjc2.18a968dd63c0cf2f namespace: kserve-ci-e2e-test resourceVersion: "20576" uid: 4eea8ef2-676a-4e43-8708-6db1a8d02dd2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:40:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-f3231} kind: Pod name: ensemble-graph-f3231-7b5968d9d6-rbjc2 namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: bf598abc-362a-4194-9c7c-8f53960c1ec8 kind: Event lastTimestamp: "2026-04-24T21:40:55Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:55Z" name: ensemble-graph-f3231-7b5968d9d6-rbjc2.18a968de06dd735a namespace: kserve-ci-e2e-test resourceVersion: "20976" uid: 57a428a0-1c94-4525-8b1e-9dba5c67f851 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-f3231-7b5968d9d6 namespace: kserve-ci-e2e-test resourceVersion: "20441" uid: 84b69648-2451-4031-8b21-889192df696c kind: Event lastTimestamp: "2026-04-24T21:40:17Z" message: 'Created pod: ensemble-graph-f3231-7b5968d9d6-rbjc2' metadata: creationTimestamp: "2026-04-24T21:40:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:17Z" name: ensemble-graph-f3231-7b5968d9d6.18a968db0bdfaaee namespace: kserve-ci-e2e-test resourceVersion: "20446" uid: 7c2a3f4d-d274-4de3-a459-fa6841b46cb1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-f3231 namespace: kserve-ci-e2e-test resourceVersion: "20439" uid: b500a115-269e-4053-8e3a-44ccf1786ed9 kind: Event lastTimestamp: "2026-04-24T21:40:17Z" message: Scaled up replica set ensemble-graph-f3231-7b5968d9d6 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:40:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:17Z" name: ensemble-graph-f3231.18a968db0b058fa4 namespace: kserve-ci-e2e-test resourceVersion: "20443" uid: e1cff1f2-853b-49b6-8af5-a73cb945ab27 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:18Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-f3231 namespace: kserve-ci-e2e-test resourceVersion: "20459" uid: 2f66fa8c-96a4-4895-b147-79f7d3b86d7a kind: Event lastTimestamp: "2026-04-24T21:40:18Z" message: InferenceGraph [ensemble-graph-f3231] is Ready metadata: creationTimestamp: "2026-04-24T21:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:40:18Z" name: ensemble-graph-f3231.18a968db10e6a79c namespace: kserve-ci-e2e-test resourceVersion: "20461" uid: 77c3b32a-7636-487e-bc4f-dfe38be12fa1 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:50:53.461885Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-12404-predictor-6cb5f67968-778ps namespace: kserve-ci-e2e-test resourceVersion: "26172" uid: 156abc99-7927-479f-8648-7631ecb91b4b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-12404-predictor-6cb5f67968-778ps to ip-10-0-132-81.ec2.internal metadata: creationTimestamp: "2026-04-24T21:50:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:50:53Z" name: error-404-isvc-12404-predictor-6cb5f67968-778ps.18a9696f01810959 namespace: kserve-ci-e2e-test resourceVersion: "26178" uid: cc66a38b-af53-4177-9ebf-628055578844 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:53Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-12404-predictor-6cb5f67968-778ps namespace: kserve-ci-e2e-test resourceVersion: "26182" uid: 156abc99-7927-479f-8648-7631ecb91b4b kind: Event lastTimestamp: "2026-04-24T21:50:53Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:50:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:50:53Z" name: error-404-isvc-12404-predictor-6cb5f67968-778ps.18a9696f1b112158 namespace: kserve-ci-e2e-test resourceVersion: "26195" uid: e0b5781a-ee1b-47df-a7fe-fe250e641838 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-12404-predictor-6cb5f67968-778ps namespace: kserve-ci-e2e-test resourceVersion: "26174" uid: 156abc99-7927-479f-8648-7631ecb91b4b kind: Event lastTimestamp: "2026-04-24T21:50:53Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1447" already present on machine metadata: creationTimestamp: "2026-04-24T21:50:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:53Z" name: error-404-isvc-12404-predictor-6cb5f67968-778ps.18a9696f1c4c8c01 namespace: kserve-ci-e2e-test resourceVersion: "26197" uid: a9148d67-a3a7-4605-abf9-d9062ccf549f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-12404-predictor-6cb5f67968-778ps namespace: kserve-ci-e2e-test resourceVersion: "26174" uid: 156abc99-7927-479f-8648-7631ecb91b4b kind: Event lastTimestamp: "2026-04-24T21:50:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:50:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:53Z" name: error-404-isvc-12404-predictor-6cb5f67968-778ps.18a9696f201b1f44 namespace: kserve-ci-e2e-test resourceVersion: "26198" uid: f74c80eb-671e-4dff-a2ad-2c21064ccac5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-12404-predictor-6cb5f67968-778ps namespace: kserve-ci-e2e-test resourceVersion: "26174" uid: 156abc99-7927-479f-8648-7631ecb91b4b kind: Event lastTimestamp: "2026-04-24T21:50:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:50:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:53Z" name: error-404-isvc-12404-predictor-6cb5f67968-778ps.18a9696f2090f784 namespace: kserve-ci-e2e-test resourceVersion: "26199" uid: 54517b47-50e4-49aa-879a-3f4f24a63ffc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-12404-predictor-6cb5f67968-778ps namespace: kserve-ci-e2e-test resourceVersion: "26174" uid: 156abc99-7927-479f-8648-7631ecb91b4b kind: Event lastTimestamp: "2026-04-24T21:50:53Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:50:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:53Z" name: error-404-isvc-12404-predictor-6cb5f67968-778ps.18a9696f209dae20 namespace: kserve-ci-e2e-test resourceVersion: "26200" uid: 77d6e1fe-bbef-460d-8bf8-e0b4e3cca7b7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-12404-predictor-6cb5f67968-778ps namespace: kserve-ci-e2e-test resourceVersion: "26174" uid: 156abc99-7927-479f-8648-7631ecb91b4b kind: Event lastTimestamp: "2026-04-24T21:50:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:54Z" name: error-404-isvc-12404-predictor-6cb5f67968-778ps.18a9696f237c4a87 namespace: kserve-ci-e2e-test resourceVersion: "26202" uid: 14ba5bdd-b2ab-492a-b28d-4bb8244fa6ed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-12404-predictor-6cb5f67968-778ps namespace: kserve-ci-e2e-test resourceVersion: "26174" uid: 156abc99-7927-479f-8648-7631ecb91b4b kind: Event lastTimestamp: "2026-04-24T21:50:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:54Z" name: error-404-isvc-12404-predictor-6cb5f67968-778ps.18a9696f23f0f824 namespace: kserve-ci-e2e-test resourceVersion: "26203" uid: cd9e38a8-5544-41d5-8353-be26e72a3170 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-12404-predictor-6cb5f67968-778ps namespace: kserve-ci-e2e-test resourceVersion: "26174" uid: 156abc99-7927-479f-8648-7631ecb91b4b kind: Event lastTimestamp: "2026-04-24T21:51:31Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:51:31Z" name: error-404-isvc-12404-predictor-6cb5f67968-778ps.18a9696f7ced4a91 namespace: kserve-ci-e2e-test resourceVersion: "26588" uid: 88fef57f-cf1a-48b0-ab3e-df849297adef reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:00:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-12404-predictor-6cb5f67968-778ps namespace: kserve-ci-e2e-test resourceVersion: "26174" uid: 156abc99-7927-479f-8648-7631ecb91b4b kind: Event lastTimestamp: "2026-04-24T22:00:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:00:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:00:08Z" name: error-404-isvc-12404-predictor-6cb5f67968-778ps.18a969f02e8468df namespace: kserve-ci-e2e-test resourceVersion: "30259" uid: 5cef1994-1b56-45dd-9d73-3f36cf1f6233 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:00:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-12404-predictor-6cb5f67968-778ps namespace: kserve-ci-e2e-test resourceVersion: "26174" uid: 156abc99-7927-479f-8648-7631ecb91b4b kind: Event lastTimestamp: "2026-04-24T22:00:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:00:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:00:08Z" name: error-404-isvc-12404-predictor-6cb5f67968-778ps.18a969f02e87b8ed namespace: kserve-ci-e2e-test resourceVersion: "30260" uid: d50bdb18-a77f-4382-8165-97f4514e3d04 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-12404-predictor-6cb5f67968 namespace: kserve-ci-e2e-test resourceVersion: "26155" uid: 8dd482f8-9458-4032-b387-7e002df41cf1 kind: Event lastTimestamp: "2026-04-24T21:50:53Z" message: 'Created pod: error-404-isvc-12404-predictor-6cb5f67968-778ps' metadata: creationTimestamp: "2026-04-24T21:50:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:53Z" name: error-404-isvc-12404-predictor-6cb5f67968.18a9696f011511b3 namespace: kserve-ci-e2e-test resourceVersion: "26175" uid: aa257209-2a30-4f56-8f4a-5f43355cd280 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-12404-predictor namespace: kserve-ci-e2e-test resourceVersion: "26152" uid: 2070f8bf-6122-47db-9126-d7403ff1c5fd kind: Event lastTimestamp: "2026-04-24T21:50:53Z" message: Scaled up replica set error-404-isvc-12404-predictor-6cb5f67968 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:50:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:53Z" name: error-404-isvc-12404-predictor.18a9696efecfdfce namespace: kserve-ci-e2e-test resourceVersion: "26161" uid: fd991f29-e701-4169-81ac-01d36be6b25d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:51:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-12404-predictor namespace: kserve-ci-e2e-test resourceVersion: "26154" uid: 490b179f-29a1-49ad-b565-6cd91d094bca kind: Event lastTimestamp: "2026-04-24T21:51:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:51:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:51:23Z" name: error-404-isvc-12404-predictor.18a969727d393b6a namespace: kserve-ci-e2e-test resourceVersion: "26526" uid: 32b4eb30-652b-44bd-81b8-d14f8f2ff80c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:51:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-12404-predictor namespace: kserve-ci-e2e-test resourceVersion: "26154" uid: 490b179f-29a1-49ad-b565-6cd91d094bca kind: Event lastTimestamp: "2026-04-24T21:51:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:51:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:51:23Z" name: error-404-isvc-12404-predictor.18a969727d91815a namespace: kserve-ci-e2e-test resourceVersion: "26527" uid: 3e13385f-734f-4d7a-aae5-71803759963c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:51:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-12404-predictor namespace: kserve-ci-e2e-test resourceVersion: "26365" uid: 490b179f-29a1-49ad-b565-6cd91d094bca kind: Event lastTimestamp: "2026-04-24T21:52:08Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:51:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:52:08Z" name: error-404-isvc-12404-predictor.18a969797b56790f namespace: kserve-ci-e2e-test resourceVersion: "26930" uid: d661555d-4775-4be9-a99f-d63f07456912 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:51:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-12404-predictor namespace: kserve-ci-e2e-test resourceVersion: "26365" uid: 490b179f-29a1-49ad-b565-6cd91d094bca kind: Event lastTimestamp: "2026-04-24T21:52:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:51:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:52:08Z" name: error-404-isvc-12404-predictor.18a969797bb2fb52 namespace: kserve-ci-e2e-test resourceVersion: "26931" uid: 9b167ee5-0050-4f2b-b0e4-dcc764c3e7a5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:50:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-12404 namespace: kserve-ci-e2e-test resourceVersion: "26136" uid: c105f355-4920-47ae-9816-61ab660fc788 kind: Event lastTimestamp: "2026-04-24T21:51:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:50:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:51:41Z" name: error-404-isvc-12404.18a9696efea85ec3 namespace: kserve-ci-e2e-test resourceVersion: "26680" uid: 29541c7d-48f0-4b5e-b87c-19d5a39e22c9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:51:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-12404 namespace: kserve-ci-e2e-test resourceVersion: "26678" uid: c105f355-4920-47ae-9816-61ab660fc788 kind: Event lastTimestamp: "2026-04-24T21:51:41Z" message: InferenceService [error-404-isvc-12404] is Ready metadata: creationTimestamp: "2026-04-24T21:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:51:41Z" name: error-404-isvc-12404.18a9697a37a770d3 namespace: kserve-ci-e2e-test resourceVersion: "26679" uid: 698a0825-5928-4dd1-9be1-ad858a0bc2fc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:30:39.102651Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3a238-predictor-6d56d496bd-kgdhd namespace: kserve-ci-e2e-test resourceVersion: "15861" uid: 2e267e3e-db1d-4581-837d-bf2a8ed36066 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3a238-predictor-6d56d496bd-kgdhd to ip-10-0-132-81.ec2.internal metadata: creationTimestamp: "2026-04-24T21:30:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:30:39Z" name: error-404-isvc-3a238-predictor-6d56d496bd-kgdhd.18a96854440db738 namespace: kserve-ci-e2e-test resourceVersion: "15865" uid: 8070e42b-d4a4-4a5e-8c6b-883fc6ef64c6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:39Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3a238-predictor-6d56d496bd-kgdhd namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 2e267e3e-db1d-4581-837d-bf2a8ed36066 kind: Event lastTimestamp: "2026-04-24T21:30:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-3a238-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:30:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:39Z" name: error-404-isvc-3a238-predictor-6d56d496bd-kgdhd.18a968544ae25643 namespace: kserve-ci-e2e-test resourceVersion: "15875" uid: 01eaa63a-a1f7-45be-bbc1-4713112ef1d2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:40Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3a238-predictor-6d56d496bd-kgdhd namespace: kserve-ci-e2e-test resourceVersion: "15873" uid: 2e267e3e-db1d-4581-837d-bf2a8ed36066 kind: Event lastTimestamp: "2026-04-24T21:30:40Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:30:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:30:40Z" name: error-404-isvc-3a238-predictor-6d56d496bd-kgdhd.18a9685480d4ccc1 namespace: kserve-ci-e2e-test resourceVersion: "15897" uid: 957f36ee-94f1-483d-8cd2-3b461d85dce6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3a238-predictor-6d56d496bd-kgdhd namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 2e267e3e-db1d-4581-837d-bf2a8ed36066 kind: Event lastTimestamp: "2026-04-24T21:30:40Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1447" already present on machine metadata: creationTimestamp: "2026-04-24T21:30:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:40Z" name: error-404-isvc-3a238-predictor-6d56d496bd-kgdhd.18a968548225d128 namespace: kserve-ci-e2e-test resourceVersion: "15900" uid: 1aec9aa8-6262-49d8-9d32-5f900f0b3a15 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3a238-predictor-6d56d496bd-kgdhd namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 2e267e3e-db1d-4581-837d-bf2a8ed36066 kind: Event lastTimestamp: "2026-04-24T21:30:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:30:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:40Z" name: error-404-isvc-3a238-predictor-6d56d496bd-kgdhd.18a9685485b54299 namespace: kserve-ci-e2e-test resourceVersion: "15903" uid: 826de84c-3ad9-44e5-bc6b-0ad98ab3adde reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3a238-predictor-6d56d496bd-kgdhd namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 2e267e3e-db1d-4581-837d-bf2a8ed36066 kind: Event lastTimestamp: "2026-04-24T21:30:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:30:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:40Z" name: error-404-isvc-3a238-predictor-6d56d496bd-kgdhd.18a9685486305aff namespace: kserve-ci-e2e-test resourceVersion: "15904" uid: df515eb9-4528-426b-9231-4316e1eec494 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3a238-predictor-6d56d496bd-kgdhd namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 2e267e3e-db1d-4581-837d-bf2a8ed36066 kind: Event lastTimestamp: "2026-04-24T21:30:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:30:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:40Z" name: error-404-isvc-3a238-predictor-6d56d496bd-kgdhd.18a96854863cd22e namespace: kserve-ci-e2e-test resourceVersion: "15905" uid: 39028ec3-7f3e-45f7-a44e-e7c1dc9f6155 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3a238-predictor-6d56d496bd-kgdhd namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 2e267e3e-db1d-4581-837d-bf2a8ed36066 kind: Event lastTimestamp: "2026-04-24T21:30:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:30:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:40Z" name: error-404-isvc-3a238-predictor-6d56d496bd-kgdhd.18a96854892f23c5 namespace: kserve-ci-e2e-test resourceVersion: "15906" uid: 3f948d36-68e5-461b-9abc-644842176539 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3a238-predictor-6d56d496bd-kgdhd namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 2e267e3e-db1d-4581-837d-bf2a8ed36066 kind: Event lastTimestamp: "2026-04-24T21:30:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:30:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:40Z" name: error-404-isvc-3a238-predictor-6d56d496bd-kgdhd.18a9685489a5802a namespace: kserve-ci-e2e-test resourceVersion: "15907" uid: 8306ec29-335c-4baf-b3d5-e9c24358a8d6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:30:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3a238-predictor-6d56d496bd-kgdhd namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 2e267e3e-db1d-4581-837d-bf2a8ed36066 kind: Event lastTimestamp: "2026-04-24T21:31:17Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:30:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:31:17Z" name: error-404-isvc-3a238-predictor-6d56d496bd-kgdhd.18a96854d3c60ed4 namespace: kserve-ci-e2e-test resourceVersion: "16290" uid: 5df8b0f2-25cf-4e2f-9613-1a9f1e86e336 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3a238-predictor-6d56d496bd-kgdhd namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 2e267e3e-db1d-4581-837d-bf2a8ed36066 kind: Event lastTimestamp: "2026-04-24T21:39:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:39:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:53Z" name: error-404-isvc-3a238-predictor-6d56d496bd-kgdhd.18a968d574c52681 namespace: kserve-ci-e2e-test resourceVersion: "20158" uid: 2f53259f-42f5-480c-a9a4-fbb33d4e8764 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3a238-predictor-6d56d496bd-kgdhd namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 2e267e3e-db1d-4581-837d-bf2a8ed36066 kind: Event lastTimestamp: "2026-04-24T21:39:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:39:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:53Z" name: error-404-isvc-3a238-predictor-6d56d496bd-kgdhd.18a968d574c5cf3d namespace: kserve-ci-e2e-test resourceVersion: "20162" uid: fd96778c-3280-4635-bbbe-32fa09960a20 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3a238-predictor-6d56d496bd namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: a7d71e07-a541-4352-9f74-aa009fc3b85d kind: Event lastTimestamp: "2026-04-24T21:30:39Z" message: 'Created pod: error-404-isvc-3a238-predictor-6d56d496bd-kgdhd' metadata: creationTimestamp: "2026-04-24T21:30:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:39Z" name: error-404-isvc-3a238-predictor-6d56d496bd.18a9685443907c22 namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: 4e49f755-a5ef-4a40-8b16-74529b1bd44c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3a238-predictor namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: 8ee094e3-db8f-4266-9236-5382e04d54a9 kind: Event lastTimestamp: "2026-04-24T21:30:39Z" message: Scaled up replica set error-404-isvc-3a238-predictor-6d56d496bd from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:30:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:39Z" name: error-404-isvc-3a238-predictor.18a9685441e622e9 namespace: kserve-ci-e2e-test resourceVersion: "15856" uid: 6f06dd07-60c1-4905-b894-c2c469849e60 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3a238-predictor namespace: kserve-ci-e2e-test resourceVersion: "15853" uid: ff7c311c-3468-41ad-b5c4-11e3d6997007 kind: Event lastTimestamp: "2026-04-24T21:30:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:30:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:54Z" name: error-404-isvc-3a238-predictor.18a96857c05e7e73 namespace: kserve-ci-e2e-test resourceVersion: "16065" uid: 8ce5fa03-6272-481c-be01-602b6007df70 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3a238-predictor namespace: kserve-ci-e2e-test resourceVersion: "15853" uid: ff7c311c-3468-41ad-b5c4-11e3d6997007 kind: Event lastTimestamp: "2026-04-24T21:30:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:30:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:54Z" name: error-404-isvc-3a238-predictor.18a96857c0e3807e namespace: kserve-ci-e2e-test resourceVersion: "16067" uid: d9b262db-6cb2-4a1f-8aba-d1f054f0339f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:31:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3a238-predictor namespace: kserve-ci-e2e-test resourceVersion: "16066" uid: ff7c311c-3468-41ad-b5c4-11e3d6997007 kind: Event lastTimestamp: "2026-04-24T21:31:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:31:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:31:54Z" name: error-404-isvc-3a238-predictor.18a9685b3f8d66f5 namespace: kserve-ci-e2e-test resourceVersion: "16634" uid: f4946dc9-0a0f-43ba-bca8-7b3c368e5ef0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:31:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3a238-predictor namespace: kserve-ci-e2e-test resourceVersion: "16066" uid: ff7c311c-3468-41ad-b5c4-11e3d6997007 kind: Event lastTimestamp: "2026-04-24T21:31:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:31:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:31:54Z" name: error-404-isvc-3a238-predictor.18a9685b3ff6462e namespace: kserve-ci-e2e-test resourceVersion: "16635" uid: 54641556-4653-43b7-bca3-7cc6cea40311 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:30:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3a238 namespace: kserve-ci-e2e-test resourceVersion: "15830" uid: 75199902-2469-4214-90bb-fb26f7c0b9af kind: Event lastTimestamp: "2026-04-24T21:31:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:30:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:31:27Z" name: error-404-isvc-3a238.18a9685441c18375 namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: 76c8e2a7-150a-4bff-9100-04fb6c67dcb5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3a238 namespace: kserve-ci-e2e-test resourceVersion: "15830" uid: 75199902-2469-4214-90bb-fb26f7c0b9af kind: Event lastTimestamp: "2026-04-24T21:30:39Z" message: 'Failed to update status for InferenceService "error-404-isvc-3a238": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3a238": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:30:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:30:39Z" name: error-404-isvc-3a238.18a9685442111920 namespace: kserve-ci-e2e-test resourceVersion: "15858" uid: 81033b74-245e-4b96-a608-4cee0a8ebb7d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3a238 namespace: kserve-ci-e2e-test resourceVersion: "15830" uid: 75199902-2469-4214-90bb-fb26f7c0b9af kind: Event lastTimestamp: "2026-04-24T21:30:39Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3a238": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:30:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:30:39Z" name: error-404-isvc-3a238.18a968544211f2c7 namespace: kserve-ci-e2e-test resourceVersion: "15859" uid: 0612c9a2-34a4-4d19-818f-6160e4073fef reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:31:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3a238 namespace: kserve-ci-e2e-test resourceVersion: "16391" uid: 75199902-2469-4214-90bb-fb26f7c0b9af kind: Event lastTimestamp: "2026-04-24T21:31:27Z" message: InferenceService [error-404-isvc-3a238] is Ready metadata: creationTimestamp: "2026-04-24T21:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:31:27Z" name: error-404-isvc-3a238.18a9685f8e6a322d namespace: kserve-ci-e2e-test resourceVersion: "16392" uid: 1030714c-610e-47ac-90a2-6ab13e1b8250 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:49:43.178826Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4efea-predictor-5fb967fbbd-hrpx8 namespace: kserve-ci-e2e-test resourceVersion: "24990" uid: 813e321c-1d89-4ab4-a19c-7a158ef051b0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4efea-predictor-5fb967fbbd-hrpx8 to ip-10-0-132-81.ec2.internal metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:49:43Z" name: error-404-isvc-4efea-predictor-5fb967fbbd-hrpx8.18a9695ea44eb3e2 namespace: kserve-ci-e2e-test resourceVersion: "24994" uid: 5773280c-b593-4c25-92f3-264151aeb74f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4efea-predictor-5fb967fbbd-hrpx8 namespace: kserve-ci-e2e-test resourceVersion: "25001" uid: 813e321c-1d89-4ab4-a19c-7a158ef051b0 kind: Event lastTimestamp: "2026-04-24T21:49:43Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:49:43Z" name: error-404-isvc-4efea-predictor-5fb967fbbd-hrpx8.18a9695ebdb8a626 namespace: kserve-ci-e2e-test resourceVersion: "25014" uid: 2cde5b2c-48a1-49e5-8abe-0094fa4c9aaa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4efea-predictor-5fb967fbbd-hrpx8 namespace: kserve-ci-e2e-test resourceVersion: "24992" uid: 813e321c-1d89-4ab4-a19c-7a158ef051b0 kind: Event lastTimestamp: "2026-04-24T21:49:43Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1447" already present on machine metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:43Z" name: error-404-isvc-4efea-predictor-5fb967fbbd-hrpx8.18a9695ebf162277 namespace: kserve-ci-e2e-test resourceVersion: "25019" uid: be1c3ddb-e31e-4385-8919-9a63a516a28b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4efea-predictor-5fb967fbbd-hrpx8 namespace: kserve-ci-e2e-test resourceVersion: "24992" uid: 813e321c-1d89-4ab4-a19c-7a158ef051b0 kind: Event lastTimestamp: "2026-04-24T21:49:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:43Z" name: error-404-isvc-4efea-predictor-5fb967fbbd-hrpx8.18a9695ec22f6cda namespace: kserve-ci-e2e-test resourceVersion: "25023" uid: 0556f2bb-c950-4a72-8fc6-0b39556992a5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4efea-predictor-5fb967fbbd-hrpx8 namespace: kserve-ci-e2e-test resourceVersion: "24992" uid: 813e321c-1d89-4ab4-a19c-7a158ef051b0 kind: Event lastTimestamp: "2026-04-24T21:49:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:43Z" name: error-404-isvc-4efea-predictor-5fb967fbbd-hrpx8.18a9695ec2a89087 namespace: kserve-ci-e2e-test resourceVersion: "25024" uid: 1abce7a2-a31f-4cb4-932a-586955a68d4f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4efea-predictor-5fb967fbbd-hrpx8 namespace: kserve-ci-e2e-test resourceVersion: "24992" uid: 813e321c-1d89-4ab4-a19c-7a158ef051b0 kind: Event lastTimestamp: "2026-04-24T21:49:43Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:43Z" name: error-404-isvc-4efea-predictor-5fb967fbbd-hrpx8.18a9695ec2b6ed38 namespace: kserve-ci-e2e-test resourceVersion: "25025" uid: 7e91f0a9-39a8-46ca-8af2-5db42c8ad7c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4efea-predictor-5fb967fbbd-hrpx8 namespace: kserve-ci-e2e-test resourceVersion: "24992" uid: 813e321c-1d89-4ab4-a19c-7a158ef051b0 kind: Event lastTimestamp: "2026-04-24T21:49:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:43Z" name: error-404-isvc-4efea-predictor-5fb967fbbd-hrpx8.18a9695ec5bbdf8a namespace: kserve-ci-e2e-test resourceVersion: "25026" uid: 25de2dd3-eca4-425e-9649-c72ea1fd83d3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4efea-predictor-5fb967fbbd-hrpx8 namespace: kserve-ci-e2e-test resourceVersion: "24992" uid: 813e321c-1d89-4ab4-a19c-7a158ef051b0 kind: Event lastTimestamp: "2026-04-24T21:49:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:43Z" name: error-404-isvc-4efea-predictor-5fb967fbbd-hrpx8.18a9695ec62f2741 namespace: kserve-ci-e2e-test resourceVersion: "25027" uid: 37c7c166-6058-49ca-bb8c-042795328035 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:49:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4efea-predictor-5fb967fbbd-hrpx8 namespace: kserve-ci-e2e-test resourceVersion: "24992" uid: 813e321c-1d89-4ab4-a19c-7a158ef051b0 kind: Event lastTimestamp: "2026-04-24T21:50:21Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:21Z" name: error-404-isvc-4efea-predictor-5fb967fbbd-hrpx8.18a9695f224348d5 namespace: kserve-ci-e2e-test resourceVersion: "25755" uid: d4a5281f-279a-4a7a-8513-ce312098ba76 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4efea-predictor-5fb967fbbd-hrpx8 namespace: kserve-ci-e2e-test resourceVersion: "24992" uid: 813e321c-1d89-4ab4-a19c-7a158ef051b0 kind: Event lastTimestamp: "2026-04-24T21:50:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:50:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:53Z" name: error-404-isvc-4efea-predictor-5fb967fbbd-hrpx8.18a9696f011e0166 namespace: kserve-ci-e2e-test resourceVersion: "26176" uid: 3f63a43a-ba29-45a5-86fb-4e6554071eeb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4efea-predictor-5fb967fbbd-hrpx8 namespace: kserve-ci-e2e-test resourceVersion: "24992" uid: 813e321c-1d89-4ab4-a19c-7a158ef051b0 kind: Event lastTimestamp: "2026-04-24T21:50:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:50:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:53Z" name: error-404-isvc-4efea-predictor-5fb967fbbd-hrpx8.18a9696f01207736 namespace: kserve-ci-e2e-test resourceVersion: "26179" uid: 6f4c6b9c-6932-4002-8e88-e3c1a46a78b5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4efea-predictor-5fb967fbbd-hrpx8 namespace: kserve-ci-e2e-test resourceVersion: "24992" uid: 813e321c-1d89-4ab4-a19c-7a158ef051b0 kind: Event lastTimestamp: "2026-04-24T21:50:56Z" message: 'Readiness probe failed: Get "https://10.133.0.35:8643/healthz": dial tcp 10.133.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:56Z" name: error-404-isvc-4efea-predictor-5fb967fbbd-hrpx8.18a9696faa7f8369 namespace: kserve-ci-e2e-test resourceVersion: "26239" uid: d55b094a-e2fe-4aab-93f4-6f02d2bfbb01 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4efea-predictor-5fb967fbbd namespace: kserve-ci-e2e-test resourceVersion: "24983" uid: a6e6f2bb-c344-46ef-98d9-fd8bb11173a1 kind: Event lastTimestamp: "2026-04-24T21:49:43Z" message: 'Created pod: error-404-isvc-4efea-predictor-5fb967fbbd-hrpx8' metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:49:43Z" name: error-404-isvc-4efea-predictor-5fb967fbbd.18a9695ea3c5a07b namespace: kserve-ci-e2e-test resourceVersion: "24991" uid: e72a6167-eb4c-4198-9ba9-8ad8f1f5aee5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4efea-predictor namespace: kserve-ci-e2e-test resourceVersion: "24981" uid: d8f5e255-9812-4cd8-bfbd-3e1976637d4d kind: Event lastTimestamp: "2026-04-24T21:49:43Z" message: Scaled up replica set error-404-isvc-4efea-predictor-5fb967fbbd from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:49:43Z" name: error-404-isvc-4efea-predictor.18a9695ea25a7db4 namespace: kserve-ci-e2e-test resourceVersion: "24985" uid: fcddc610-5156-4fb7-b0b4-2c17e947b1b1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:49:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4efea-predictor namespace: kserve-ci-e2e-test resourceVersion: "24982" uid: 4921f73a-3be9-4a1b-b517-4bc27702adc7 kind: Event lastTimestamp: "2026-04-24T21:50:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:49:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:28Z" name: error-404-isvc-4efea-predictor.18a9696220eca61c namespace: kserve-ci-e2e-test resourceVersion: "25826" uid: a3ad9b44-ea68-4542-9a1e-31fdd27ab0ce reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:49:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4efea-predictor namespace: kserve-ci-e2e-test resourceVersion: "24982" uid: 4921f73a-3be9-4a1b-b517-4bc27702adc7 kind: Event lastTimestamp: "2026-04-24T21:50:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:49:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:28Z" name: error-404-isvc-4efea-predictor.18a96962214d213c namespace: kserve-ci-e2e-test resourceVersion: "25827" uid: 242f075b-3d84-481f-9bfc-57b28bf2266b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4efea-predictor namespace: kserve-ci-e2e-test resourceVersion: "25448" uid: 4921f73a-3be9-4a1b-b517-4bc27702adc7 kind: Event lastTimestamp: "2026-04-24T21:50:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:50:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:43Z" name: error-404-isvc-4efea-predictor.18a9696c9e23f723 namespace: kserve-ci-e2e-test resourceVersion: "25985" uid: 7553a46c-60e2-4b00-9229-41245a31e04d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4efea-predictor namespace: kserve-ci-e2e-test resourceVersion: "25448" uid: 4921f73a-3be9-4a1b-b517-4bc27702adc7 kind: Event lastTimestamp: "2026-04-24T21:50:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:50:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:43Z" name: error-404-isvc-4efea-predictor.18a9696c9ea43450 namespace: kserve-ci-e2e-test resourceVersion: "25989" uid: d561c3d2-0ef7-4b4d-9186-9abcb576ef28 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4efea namespace: kserve-ci-e2e-test resourceVersion: "24963" uid: f091e1a8-1d10-43c2-ae73-b402d76edd2e kind: Event lastTimestamp: "2026-04-24T21:50:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:31Z" name: error-404-isvc-4efea.18a9695ea238b224 namespace: kserve-ci-e2e-test resourceVersion: "25858" uid: b6d70393-08c4-41f2-8ab4-3fe6137c8ed9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4efea namespace: kserve-ci-e2e-test resourceVersion: "24963" uid: f091e1a8-1d10-43c2-ae73-b402d76edd2e kind: Event lastTimestamp: "2026-04-24T21:49:43Z" message: 'Failed to update status for InferenceService "error-404-isvc-4efea": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4efea": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:49:43Z" name: error-404-isvc-4efea.18a9695ea2a2bf44 namespace: kserve-ci-e2e-test resourceVersion: "24987" uid: 453bd8c5-64f4-41d7-8e4f-71d9d7491a9e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4efea namespace: kserve-ci-e2e-test resourceVersion: "24963" uid: f091e1a8-1d10-43c2-ae73-b402d76edd2e kind: Event lastTimestamp: "2026-04-24T21:49:43Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4efea": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:49:43Z" name: error-404-isvc-4efea.18a9695ea2a31bbc namespace: kserve-ci-e2e-test resourceVersion: "24988" uid: 29de5c40-6593-44d5-b59f-a6c2499eaf07 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4efea namespace: kserve-ci-e2e-test resourceVersion: "25856" uid: f091e1a8-1d10-43c2-ae73-b402d76edd2e kind: Event lastTimestamp: "2026-04-24T21:50:31Z" message: InferenceService [error-404-isvc-4efea] is Ready metadata: creationTimestamp: "2026-04-24T21:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:31Z" name: error-404-isvc-4efea.18a96969dce3e773 namespace: kserve-ci-e2e-test resourceVersion: "25857" uid: 189fcb8d-cf6b-4d4a-bf07-c1f616b2cf7a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:41:04.234667Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-64806-predictor-5cd95b49d6-g7v8z namespace: kserve-ci-e2e-test resourceVersion: "21140" uid: 0b157333-a6fe-4cb3-9680-aa9b1570c066 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-64806-predictor-5cd95b49d6-g7v8z to ip-10-0-132-81.ec2.internal metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:41:04Z" name: error-404-isvc-64806-predictor-5cd95b49d6-g7v8z.18a968e5d0d33ac0 namespace: kserve-ci-e2e-test resourceVersion: "21144" uid: 3722e924-27a3-4acc-8457-6c8c848e4875 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-64806-predictor-5cd95b49d6-g7v8z namespace: kserve-ci-e2e-test resourceVersion: "21150" uid: 0b157333-a6fe-4cb3-9680-aa9b1570c066 kind: Event lastTimestamp: "2026-04-24T21:41:04Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:41:04Z" name: error-404-isvc-64806-predictor-5cd95b49d6-g7v8z.18a968e5ea6839fb namespace: kserve-ci-e2e-test resourceVersion: "21164" uid: ea798c8e-df5e-46e1-a218-b56eed41d2b5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-64806-predictor-5cd95b49d6-g7v8z namespace: kserve-ci-e2e-test resourceVersion: "21142" uid: 0b157333-a6fe-4cb3-9680-aa9b1570c066 kind: Event lastTimestamp: "2026-04-24T21:41:04Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1447" already present on machine metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:04Z" name: error-404-isvc-64806-predictor-5cd95b49d6-g7v8z.18a968e5ebe38b8d namespace: kserve-ci-e2e-test resourceVersion: "21168" uid: 135c0717-02f5-43ef-bea7-53dd5a01aeb6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-64806-predictor-5cd95b49d6-g7v8z namespace: kserve-ci-e2e-test resourceVersion: "21142" uid: 0b157333-a6fe-4cb3-9680-aa9b1570c066 kind: Event lastTimestamp: "2026-04-24T21:41:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:04Z" name: error-404-isvc-64806-predictor-5cd95b49d6-g7v8z.18a968e5ef465560 namespace: kserve-ci-e2e-test resourceVersion: "21169" uid: c01c0a31-b338-44b9-8036-44a4a2d5dee5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-64806-predictor-5cd95b49d6-g7v8z namespace: kserve-ci-e2e-test resourceVersion: "21142" uid: 0b157333-a6fe-4cb3-9680-aa9b1570c066 kind: Event lastTimestamp: "2026-04-24T21:41:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:04Z" name: error-404-isvc-64806-predictor-5cd95b49d6-g7v8z.18a968e5efcdc3e2 namespace: kserve-ci-e2e-test resourceVersion: "21170" uid: b97a6a77-6a1f-4dc6-9791-f33e1bb91928 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-64806-predictor-5cd95b49d6-g7v8z namespace: kserve-ci-e2e-test resourceVersion: "21142" uid: 0b157333-a6fe-4cb3-9680-aa9b1570c066 kind: Event lastTimestamp: "2026-04-24T21:41:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:04Z" name: error-404-isvc-64806-predictor-5cd95b49d6-g7v8z.18a968e5efdb4db0 namespace: kserve-ci-e2e-test resourceVersion: "21171" uid: 008ae9df-54cb-415a-9c4e-6c9137f1974f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-64806-predictor-5cd95b49d6-g7v8z namespace: kserve-ci-e2e-test resourceVersion: "21142" uid: 0b157333-a6fe-4cb3-9680-aa9b1570c066 kind: Event lastTimestamp: "2026-04-24T21:41:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:04Z" name: error-404-isvc-64806-predictor-5cd95b49d6-g7v8z.18a968e5f294825e namespace: kserve-ci-e2e-test resourceVersion: "21172" uid: 7f730c8b-6cc9-4bf1-bf8f-7fbb90d86104 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-64806-predictor-5cd95b49d6-g7v8z namespace: kserve-ci-e2e-test resourceVersion: "21142" uid: 0b157333-a6fe-4cb3-9680-aa9b1570c066 kind: Event lastTimestamp: "2026-04-24T21:41:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:04Z" name: error-404-isvc-64806-predictor-5cd95b49d6-g7v8z.18a968e5f30e46a5 namespace: kserve-ci-e2e-test resourceVersion: "21173" uid: 983edb4a-80da-4700-8fb3-c5515565ce41 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:41:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-64806-predictor-5cd95b49d6-g7v8z namespace: kserve-ci-e2e-test resourceVersion: "21142" uid: 0b157333-a6fe-4cb3-9680-aa9b1570c066 kind: Event lastTimestamp: "2026-04-24T21:50:22Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:41:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:22Z" name: error-404-isvc-64806-predictor-5cd95b49d6-g7v8z.18a968e65d0513c0 namespace: kserve-ci-e2e-test resourceVersion: "25773" uid: 3585d9e8-f64c-4ad4-90a3-419600d4ef61 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-64806-predictor-5cd95b49d6-g7v8z namespace: kserve-ci-e2e-test resourceVersion: "21142" uid: 0b157333-a6fe-4cb3-9680-aa9b1570c066 kind: Event lastTimestamp: "2026-04-24T21:50:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:50:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:19Z" name: error-404-isvc-64806-predictor-5cd95b49d6-g7v8z.18a96967014f7f5a namespace: kserve-ci-e2e-test resourceVersion: "25689" uid: 107d7e3f-ecd8-42d0-871c-6c1bc03bdb9f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-64806-predictor-5cd95b49d6-g7v8z namespace: kserve-ci-e2e-test resourceVersion: "21142" uid: 0b157333-a6fe-4cb3-9680-aa9b1570c066 kind: Event lastTimestamp: "2026-04-24T21:50:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:50:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:19Z" name: error-404-isvc-64806-predictor-5cd95b49d6-g7v8z.18a969670152b28e namespace: kserve-ci-e2e-test resourceVersion: "25692" uid: 019135a6-4a8c-428b-b92b-78e17796f5b4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-64806-predictor-5cd95b49d6-g7v8z namespace: kserve-ci-e2e-test resourceVersion: "21142" uid: 0b157333-a6fe-4cb3-9680-aa9b1570c066 kind: Event lastTimestamp: "2026-04-24T21:50:22Z" message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": dial tcp 10.133.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:50:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:22Z" name: error-404-isvc-64806-predictor-5cd95b49d6-g7v8z.18a96967d17c2c6b namespace: kserve-ci-e2e-test resourceVersion: "25772" uid: 5a43fa0d-f670-4f0a-8de4-ec2e4531ed1d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-64806-predictor-5cd95b49d6 namespace: kserve-ci-e2e-test resourceVersion: "21133" uid: cfd115ff-6dbc-4439-be92-bb726fe9063e kind: Event lastTimestamp: "2026-04-24T21:41:04Z" message: 'Created pod: error-404-isvc-64806-predictor-5cd95b49d6-g7v8z' metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:04Z" name: error-404-isvc-64806-predictor-5cd95b49d6.18a968e5d06c0e54 namespace: kserve-ci-e2e-test resourceVersion: "21141" uid: f5bab8d0-ad21-4b6a-b81d-342f726e16fa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-64806-predictor namespace: kserve-ci-e2e-test resourceVersion: "21131" uid: d6b7bcab-21fd-4751-bdc7-825975478dd1 kind: Event lastTimestamp: "2026-04-24T21:41:04Z" message: Scaled up replica set error-404-isvc-64806-predictor-5cd95b49d6 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:04Z" name: error-404-isvc-64806-predictor.18a968e5cecf9d0c namespace: kserve-ci-e2e-test resourceVersion: "21135" uid: d052fe68-d031-4f8b-bfc6-c8677e184fa0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-64806-predictor namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 175b2899-947a-4753-ae38-4a45aff9b0c2 kind: Event lastTimestamp: "2026-04-24T21:41:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:19Z" name: error-404-isvc-64806-predictor.18a968e94d3f7c38 namespace: kserve-ci-e2e-test resourceVersion: "21329" uid: 79c3d321-6907-4c80-9f36-84d1e5221077 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-64806-predictor namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 175b2899-947a-4753-ae38-4a45aff9b0c2 kind: Event lastTimestamp: "2026-04-24T21:41:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:19Z" name: error-404-isvc-64806-predictor.18a968e94d97efe0 namespace: kserve-ci-e2e-test resourceVersion: "21330" uid: e02532c6-607a-4132-8641-1774f5b347a5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:41:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-64806-predictor namespace: kserve-ci-e2e-test resourceVersion: "21328" uid: 175b2899-947a-4753-ae38-4a45aff9b0c2 kind: Event lastTimestamp: "2026-04-24T21:42:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:41:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:42:19Z" name: error-404-isvc-64806-predictor.18a968ecccc489bf namespace: kserve-ci-e2e-test resourceVersion: "21887" uid: f3ce91eb-f262-44f3-a5b5-25cdb2cb83fd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:41:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-64806-predictor namespace: kserve-ci-e2e-test resourceVersion: "21328" uid: 175b2899-947a-4753-ae38-4a45aff9b0c2 kind: Event lastTimestamp: "2026-04-24T21:42:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:41:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:42:19Z" name: error-404-isvc-64806-predictor.18a968eccd30855c namespace: kserve-ci-e2e-test resourceVersion: "21888" uid: f7b68799-b26e-40a7-960a-7055b9001091 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-64806 namespace: kserve-ci-e2e-test resourceVersion: "21118" uid: 4482f13f-0972-48d7-a5e1-7457f7cfd214 kind: Event lastTimestamp: "2026-04-24T21:41:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:41:52Z" name: error-404-isvc-64806.18a968e5cea84518 namespace: kserve-ci-e2e-test resourceVersion: "21651" uid: 8f005e95-47c2-459e-9dc6-77b46d15a007 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-64806 namespace: kserve-ci-e2e-test resourceVersion: "21118" uid: 4482f13f-0972-48d7-a5e1-7457f7cfd214 kind: Event lastTimestamp: "2026-04-24T21:41:04Z" message: 'Failed to update status for InferenceService "error-404-isvc-64806": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-64806": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:41:04Z" name: error-404-isvc-64806.18a968e5cf07cc67 namespace: kserve-ci-e2e-test resourceVersion: "21137" uid: 46d6d01b-281e-4837-8ff0-393f1b9afbf4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-64806 namespace: kserve-ci-e2e-test resourceVersion: "21118" uid: 4482f13f-0972-48d7-a5e1-7457f7cfd214 kind: Event lastTimestamp: "2026-04-24T21:41:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-64806": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:41:04Z" name: error-404-isvc-64806.18a968e5cf0825ac namespace: kserve-ci-e2e-test resourceVersion: "21138" uid: be1ed4c6-83ed-422b-aa9f-9791e0dd4d0d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-64806 namespace: kserve-ci-e2e-test resourceVersion: "21649" uid: 4482f13f-0972-48d7-a5e1-7457f7cfd214 kind: Event lastTimestamp: "2026-04-24T21:41:52Z" message: InferenceService [error-404-isvc-64806] is Ready metadata: creationTimestamp: "2026-04-24T21:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:41:52Z" name: error-404-isvc-64806.18a968f1182eecff namespace: kserve-ci-e2e-test resourceVersion: "21650" uid: 1043847e-7a36-44e9-852d-5b406ea37d80 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:40:28.317264Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-75b0d-predictor-758f7456fd-b6kvp namespace: kserve-ci-e2e-test resourceVersion: "20652" uid: 105dc369-8ab5-42ab-bf68-ea4f803de9f1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-75b0d-predictor-758f7456fd-b6kvp to ip-10-0-132-81.ec2.internal metadata: creationTimestamp: "2026-04-24T21:40:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:40:28Z" name: error-404-isvc-75b0d-predictor-758f7456fd-b6kvp.18a968dd73fb2e50 namespace: kserve-ci-e2e-test resourceVersion: "20656" uid: 08751392-443f-4990-b172-dc6fc301a879 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:28Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-75b0d-predictor-758f7456fd-b6kvp namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: 105dc369-8ab5-42ab-bf68-ea4f803de9f1 kind: Event lastTimestamp: "2026-04-24T21:40:28Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:40:28Z" name: error-404-isvc-75b0d-predictor-758f7456fd-b6kvp.18a968dd8dd2a955 namespace: kserve-ci-e2e-test resourceVersion: "20672" uid: f8c76e46-cd82-4c13-a978-2398560d6db7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-75b0d-predictor-758f7456fd-b6kvp namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 105dc369-8ab5-42ab-bf68-ea4f803de9f1 kind: Event lastTimestamp: "2026-04-24T21:40:28Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1447" already present on machine metadata: creationTimestamp: "2026-04-24T21:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:28Z" name: error-404-isvc-75b0d-predictor-758f7456fd-b6kvp.18a968dd8f309931 namespace: kserve-ci-e2e-test resourceVersion: "20674" uid: 48dafb8b-0a25-4300-8a3e-05a0f9b5fbe4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-75b0d-predictor-758f7456fd-b6kvp namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 105dc369-8ab5-42ab-bf68-ea4f803de9f1 kind: Event lastTimestamp: "2026-04-24T21:40:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:28Z" name: error-404-isvc-75b0d-predictor-758f7456fd-b6kvp.18a968dd91d7752f namespace: kserve-ci-e2e-test resourceVersion: "20676" uid: 7430b4e3-d491-4e88-9b09-a22a199e2eb0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-75b0d-predictor-758f7456fd-b6kvp namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 105dc369-8ab5-42ab-bf68-ea4f803de9f1 kind: Event lastTimestamp: "2026-04-24T21:40:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:28Z" name: error-404-isvc-75b0d-predictor-758f7456fd-b6kvp.18a968dd924e0910 namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: 16e62a7a-69c7-45a0-88ed-5d006136fb88 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-75b0d-predictor-758f7456fd-b6kvp namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 105dc369-8ab5-42ab-bf68-ea4f803de9f1 kind: Event lastTimestamp: "2026-04-24T21:40:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:28Z" name: error-404-isvc-75b0d-predictor-758f7456fd-b6kvp.18a968dd925c4ce6 namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: 6eed63ba-9202-4721-9d81-adcdde1deaaf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-75b0d-predictor-758f7456fd-b6kvp namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 105dc369-8ab5-42ab-bf68-ea4f803de9f1 kind: Event lastTimestamp: "2026-04-24T21:40:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:28Z" name: error-404-isvc-75b0d-predictor-758f7456fd-b6kvp.18a968dd9505d8f6 namespace: kserve-ci-e2e-test resourceVersion: "20680" uid: f715696a-4288-47c7-8320-36037f32918d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-75b0d-predictor-758f7456fd-b6kvp namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 105dc369-8ab5-42ab-bf68-ea4f803de9f1 kind: Event lastTimestamp: "2026-04-24T21:40:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:28Z" name: error-404-isvc-75b0d-predictor-758f7456fd-b6kvp.18a968dd957e17f5 namespace: kserve-ci-e2e-test resourceVersion: "20681" uid: 7b824ec4-81b9-4309-9051-fc1fbae87c4f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:40:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-75b0d-predictor-758f7456fd-b6kvp namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 105dc369-8ab5-42ab-bf68-ea4f803de9f1 kind: Event lastTimestamp: "2026-04-24T21:41:06Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:06Z" name: error-404-isvc-75b0d-predictor-758f7456fd-b6kvp.18a968ddf3efed89 namespace: kserve-ci-e2e-test resourceVersion: "21198" uid: eba5e58d-bdd4-45ff-a2df-c0081cd1442f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-75b0d-predictor-758f7456fd-b6kvp namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 105dc369-8ab5-42ab-bf68-ea4f803de9f1 kind: Event lastTimestamp: "2026-04-24T21:49:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:43Z" name: error-404-isvc-75b0d-predictor-758f7456fd-b6kvp.18a9695e9e55e41b namespace: kserve-ci-e2e-test resourceVersion: "24962" uid: e728b886-fa2f-4e4f-a3fa-ac4c9d43c271 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-75b0d-predictor-758f7456fd-b6kvp namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 105dc369-8ab5-42ab-bf68-ea4f803de9f1 kind: Event lastTimestamp: "2026-04-24T21:49:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:43Z" name: error-404-isvc-75b0d-predictor-758f7456fd-b6kvp.18a9695e9e56a576 namespace: kserve-ci-e2e-test resourceVersion: "24964" uid: 116a24c2-d3a4-44b9-9cc4-7866e5b83d40 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-75b0d-predictor-758f7456fd namespace: kserve-ci-e2e-test resourceVersion: "20645" uid: 31cb5f33-d157-46b2-aed1-b8a0773e06c0 kind: Event lastTimestamp: "2026-04-24T21:40:28Z" message: 'Created pod: error-404-isvc-75b0d-predictor-758f7456fd-b6kvp' metadata: creationTimestamp: "2026-04-24T21:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:28Z" name: error-404-isvc-75b0d-predictor-758f7456fd.18a968dd7361b79c namespace: kserve-ci-e2e-test resourceVersion: "20653" uid: 121acd0f-9378-49f8-b158-895da6ee2b62 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-75b0d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20642" uid: 66d355f2-65ef-43c5-a176-45db0dcd65a0 kind: Event lastTimestamp: "2026-04-24T21:40:28Z" message: Scaled up replica set error-404-isvc-75b0d-predictor-758f7456fd from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:28Z" name: error-404-isvc-75b0d-predictor.18a968dd718f7988 namespace: kserve-ci-e2e-test resourceVersion: "20647" uid: a992a0c4-2708-46ba-be03-48af149e7e65 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:40:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-75b0d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20644" uid: ecca4a05-aa60-4322-873f-6b10bfdc7709 kind: Event lastTimestamp: "2026-04-24T21:40:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:40:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:58Z" name: error-404-isvc-75b0d-predictor.18a968e0eff1d106 namespace: kserve-ci-e2e-test resourceVersion: "20997" uid: ece5fd06-588f-450a-85a2-a91d6ace469e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:40:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-75b0d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20644" uid: ecca4a05-aa60-4322-873f-6b10bfdc7709 kind: Event lastTimestamp: "2026-04-24T21:40:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:40:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:58Z" name: error-404-isvc-75b0d-predictor.18a968e0f04f44cc namespace: kserve-ci-e2e-test resourceVersion: "20998" uid: c1cf575b-0127-44de-8759-fc6cb6fc8ffb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:41:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-75b0d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20842" uid: ecca4a05-aa60-4322-873f-6b10bfdc7709 kind: Event lastTimestamp: "2026-04-24T21:41:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:41:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:43Z" name: error-404-isvc-75b0d-predictor.18a968e7edaec0f8 namespace: kserve-ci-e2e-test resourceVersion: "21565" uid: ae847c5d-8126-4b80-aada-6e6f91a009bb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:41:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-75b0d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20842" uid: ecca4a05-aa60-4322-873f-6b10bfdc7709 kind: Event lastTimestamp: "2026-04-24T21:41:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:41:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:43Z" name: error-404-isvc-75b0d-predictor.18a968e7ee0ddad3 namespace: kserve-ci-e2e-test resourceVersion: "21566" uid: aeb018b7-2b19-4f31-8d74-113934f2c9d9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:40:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-75b0d namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: ae3fd40c-2ac8-4d09-8813-7de06f056f76 kind: Event lastTimestamp: "2026-04-24T21:41:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:41:16Z" name: error-404-isvc-75b0d.18a968dd716d245d namespace: kserve-ci-e2e-test resourceVersion: "21309" uid: c7a76123-fddb-408f-ab5d-e7d9484263c7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-75b0d namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: ae3fd40c-2ac8-4d09-8813-7de06f056f76 kind: Event lastTimestamp: "2026-04-24T21:40:28Z" message: 'Failed to update status for InferenceService "error-404-isvc-75b0d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-75b0d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:40:28Z" name: error-404-isvc-75b0d.18a968dd71d459bc namespace: kserve-ci-e2e-test resourceVersion: "20649" uid: d3c88d50-5ac0-4adf-84c2-3acc8b9650bd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-75b0d namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: ae3fd40c-2ac8-4d09-8813-7de06f056f76 kind: Event lastTimestamp: "2026-04-24T21:40:28Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-75b0d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:40:28Z" name: error-404-isvc-75b0d.18a968dd71d4c444 namespace: kserve-ci-e2e-test resourceVersion: "20650" uid: 659d1815-6091-430c-b3d1-a82b646a1c9a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-75b0d namespace: kserve-ci-e2e-test resourceVersion: "21306" uid: ae3fd40c-2ac8-4d09-8813-7de06f056f76 kind: Event lastTimestamp: "2026-04-24T21:41:16Z" message: InferenceService [error-404-isvc-75b0d] is Ready metadata: creationTimestamp: "2026-04-24T21:41:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:41:16Z" name: error-404-isvc-75b0d.18a968e8af20f209 namespace: kserve-ci-e2e-test resourceVersion: "21308" uid: dc4815fa-d6be-44c2-8287-83bb84718e69 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:50:19.143804Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-98aac-predictor-5678f7cc74-45scf namespace: kserve-ci-e2e-test resourceVersion: "25697" uid: f69bd325-9fbb-4e92-bc0d-942db20130b2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-98aac-predictor-5678f7cc74-45scf to ip-10-0-132-81.ec2.internal metadata: creationTimestamp: "2026-04-24T21:50:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:50:19Z" name: error-404-isvc-98aac-predictor-5678f7cc74-45scf.18a9696703fcb76c namespace: kserve-ci-e2e-test resourceVersion: "25701" uid: 87c39eb4-1e72-409a-840f-f33292c1acda reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:19Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-98aac-predictor-5678f7cc74-45scf namespace: kserve-ci-e2e-test resourceVersion: "25699" uid: f69bd325-9fbb-4e92-bc0d-942db20130b2 kind: Event lastTimestamp: "2026-04-24T21:50:19Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-98aac-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:50:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:19Z" name: error-404-isvc-98aac-predictor-5678f7cc74-45scf.18a969670c5742f6 namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: 58b12cb8-4b61-493b-99e7-a39f8f67cf21 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:20Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-98aac-predictor-5678f7cc74-45scf namespace: kserve-ci-e2e-test resourceVersion: "25707" uid: f69bd325-9fbb-4e92-bc0d-942db20130b2 kind: Event lastTimestamp: "2026-04-24T21:50:20Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:50:20Z" name: error-404-isvc-98aac-predictor-5678f7cc74-45scf.18a96967413b23e6 namespace: kserve-ci-e2e-test resourceVersion: "25730" uid: 3c859c82-83b8-4c66-b887-72f34ba4ff4e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-98aac-predictor-5678f7cc74-45scf namespace: kserve-ci-e2e-test resourceVersion: "25699" uid: f69bd325-9fbb-4e92-bc0d-942db20130b2 kind: Event lastTimestamp: "2026-04-24T21:50:20Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1447" already present on machine metadata: creationTimestamp: "2026-04-24T21:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:20Z" name: error-404-isvc-98aac-predictor-5678f7cc74-45scf.18a96967429d9b1a namespace: kserve-ci-e2e-test resourceVersion: "25732" uid: 2c8c0d4f-a83b-425d-a69c-d56636b36264 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-98aac-predictor-5678f7cc74-45scf namespace: kserve-ci-e2e-test resourceVersion: "25699" uid: f69bd325-9fbb-4e92-bc0d-942db20130b2 kind: Event lastTimestamp: "2026-04-24T21:50:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:20Z" name: error-404-isvc-98aac-predictor-5678f7cc74-45scf.18a9696745f41f43 namespace: kserve-ci-e2e-test resourceVersion: "25736" uid: 038bf829-b5d7-4973-8ddb-ac2959844b70 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-98aac-predictor-5678f7cc74-45scf namespace: kserve-ci-e2e-test resourceVersion: "25699" uid: f69bd325-9fbb-4e92-bc0d-942db20130b2 kind: Event lastTimestamp: "2026-04-24T21:50:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:20Z" name: error-404-isvc-98aac-predictor-5678f7cc74-45scf.18a96967466d9479 namespace: kserve-ci-e2e-test resourceVersion: "25737" uid: 0d5b6b58-a67f-45bf-84c7-ef5a8e25f9bd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-98aac-predictor-5678f7cc74-45scf namespace: kserve-ci-e2e-test resourceVersion: "25699" uid: f69bd325-9fbb-4e92-bc0d-942db20130b2 kind: Event lastTimestamp: "2026-04-24T21:50:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:20Z" name: error-404-isvc-98aac-predictor-5678f7cc74-45scf.18a96967467a788e namespace: kserve-ci-e2e-test resourceVersion: "25738" uid: 2e686955-b84c-45b2-9ee3-53454325ae67 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-98aac-predictor-5678f7cc74-45scf namespace: kserve-ci-e2e-test resourceVersion: "25699" uid: f69bd325-9fbb-4e92-bc0d-942db20130b2 kind: Event lastTimestamp: "2026-04-24T21:50:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:20Z" name: error-404-isvc-98aac-predictor-5678f7cc74-45scf.18a969674a3c5931 namespace: kserve-ci-e2e-test resourceVersion: "25739" uid: c61d762a-af55-4a1a-b5ac-33e0707f3434 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-98aac-predictor-5678f7cc74-45scf namespace: kserve-ci-e2e-test resourceVersion: "25699" uid: f69bd325-9fbb-4e92-bc0d-942db20130b2 kind: Event lastTimestamp: "2026-04-24T21:50:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:20Z" name: error-404-isvc-98aac-predictor-5678f7cc74-45scf.18a969674ab10ebf namespace: kserve-ci-e2e-test resourceVersion: "25740" uid: f34290d4-9824-493e-bedf-37092bf98767 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:50:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-98aac-predictor-5678f7cc74-45scf namespace: kserve-ci-e2e-test resourceVersion: "25699" uid: f69bd325-9fbb-4e92-bc0d-942db20130b2 kind: Event lastTimestamp: "2026-04-24T21:50:57Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:50:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:57Z" name: error-404-isvc-98aac-predictor-5678f7cc74-45scf.18a969678b2f522f namespace: kserve-ci-e2e-test resourceVersion: "26257" uid: 824d332d-545e-42b2-8365-58cf9bd0c467 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-98aac-predictor-5678f7cc74-45scf namespace: kserve-ci-e2e-test resourceVersion: "25699" uid: f69bd325-9fbb-4e92-bc0d-942db20130b2 kind: Event lastTimestamp: "2026-04-24T22:07:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:07:38Z" name: error-404-isvc-98aac-predictor-5678f7cc74-45scf.18a96a590453bbc2 namespace: kserve-ci-e2e-test resourceVersion: "33408" uid: 209896d2-cc38-4990-b477-a68dde3589bf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-98aac-predictor-5678f7cc74-45scf namespace: kserve-ci-e2e-test resourceVersion: "25699" uid: f69bd325-9fbb-4e92-bc0d-942db20130b2 kind: Event lastTimestamp: "2026-04-24T22:07:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:07:38Z" name: error-404-isvc-98aac-predictor-5678f7cc74-45scf.18a96a5904542e66 namespace: kserve-ci-e2e-test resourceVersion: "33409" uid: a6445a34-4d53-41d3-913c-2c0f4557a41c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-98aac-predictor-5678f7cc74 namespace: kserve-ci-e2e-test resourceVersion: "25688" uid: 03e4d5f0-39a8-4edb-9c29-097985881b5c kind: Event lastTimestamp: "2026-04-24T21:50:19Z" message: 'Created pod: error-404-isvc-98aac-predictor-5678f7cc74-45scf' metadata: creationTimestamp: "2026-04-24T21:50:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:19Z" name: error-404-isvc-98aac-predictor-5678f7cc74.18a969670385581e namespace: kserve-ci-e2e-test resourceVersion: "25698" uid: 7d673836-68cf-47c4-87ce-47ef2f358ca6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-98aac-predictor namespace: kserve-ci-e2e-test resourceVersion: "25686" uid: 703feae6-93fa-44ba-9b45-b89ce27a9a15 kind: Event lastTimestamp: "2026-04-24T21:50:19Z" message: Scaled up replica set error-404-isvc-98aac-predictor-5678f7cc74 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:50:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:19Z" name: error-404-isvc-98aac-predictor.18a9696701a741b0 namespace: kserve-ci-e2e-test resourceVersion: "25691" uid: 3d873eb8-05d3-4ae5-84a1-6fc0bdd2aa68 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:50:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-98aac-predictor namespace: kserve-ci-e2e-test resourceVersion: "25687" uid: bdaa4233-c33a-408a-877d-fee77f8eb954 kind: Event lastTimestamp: "2026-04-24T21:51:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:50:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:51:19Z" name: error-404-isvc-98aac-predictor.18a9696a80279733 namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 4b618cba-ede3-4210-a240-0086fdabfc04 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:50:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-98aac-predictor namespace: kserve-ci-e2e-test resourceVersion: "25687" uid: bdaa4233-c33a-408a-877d-fee77f8eb954 kind: Event lastTimestamp: "2026-04-24T21:51:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:50:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:51:19Z" name: error-404-isvc-98aac-predictor.18a9696a808c1501 namespace: kserve-ci-e2e-test resourceVersion: "26477" uid: 8a7d2927-78e9-40da-b1f0-ada95b78d3fa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:51:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-98aac-predictor namespace: kserve-ci-e2e-test resourceVersion: "25889" uid: bdaa4233-c33a-408a-877d-fee77f8eb954 kind: Event lastTimestamp: "2026-04-24T21:51:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:51:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:51:34Z" name: error-404-isvc-98aac-predictor.18a969787bcdd41f namespace: kserve-ci-e2e-test resourceVersion: "26615" uid: 4711fc46-2b24-434b-889f-d2cf9d6eb8e6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:51:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-98aac-predictor namespace: kserve-ci-e2e-test resourceVersion: "25889" uid: bdaa4233-c33a-408a-877d-fee77f8eb954 kind: Event lastTimestamp: "2026-04-24T21:51:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:51:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:51:34Z" name: error-404-isvc-98aac-predictor.18a969787c3b827e namespace: kserve-ci-e2e-test resourceVersion: "26616" uid: d254cbca-c189-44c5-8963-4d12484e24e4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:50:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-98aac namespace: kserve-ci-e2e-test resourceVersion: "25658" uid: a1924171-c4c1-4839-bcdb-870337cf94b9 kind: Event lastTimestamp: "2026-04-24T21:51:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:50:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:51:07Z" name: error-404-isvc-98aac.18a969670185e020 namespace: kserve-ci-e2e-test resourceVersion: "26346" uid: a3f635d3-c0a3-487a-b340-2de8eae5a159 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-98aac namespace: kserve-ci-e2e-test resourceVersion: "25658" uid: a1924171-c4c1-4839-bcdb-870337cf94b9 kind: Event lastTimestamp: "2026-04-24T21:50:19Z" message: 'Failed to update status for InferenceService "error-404-isvc-98aac": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-98aac": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:50:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:19Z" name: error-404-isvc-98aac.18a9696701f9b879 namespace: kserve-ci-e2e-test resourceVersion: "25694" uid: 20d660db-a2e6-4c01-89c9-5f3435054872 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-98aac namespace: kserve-ci-e2e-test resourceVersion: "25658" uid: a1924171-c4c1-4839-bcdb-870337cf94b9 kind: Event lastTimestamp: "2026-04-24T21:50:19Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-98aac": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:50:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:19Z" name: error-404-isvc-98aac.18a9696701fa125e namespace: kserve-ci-e2e-test resourceVersion: "25695" uid: 6d96224a-ce15-44eb-8610-df2499cf2cc2 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:51:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-98aac namespace: kserve-ci-e2e-test resourceVersion: "26344" uid: a1924171-c4c1-4839-bcdb-870337cf94b9 kind: Event lastTimestamp: "2026-04-24T21:51:07Z" message: InferenceService [error-404-isvc-98aac] is Ready metadata: creationTimestamp: "2026-04-24T21:51:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:51:07Z" name: error-404-isvc-98aac.18a96972460150a8 namespace: kserve-ci-e2e-test resourceVersion: "26345" uid: 3b4f3bd2-4abf-442e-9912-afcd6defcb27 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:30:03.630117Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk namespace: kserve-ci-e2e-test resourceVersion: "15359" uid: 319d9352-b194-4a33-83c2-3906eaa110fd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b868d-predictor-5b5b76b898-87zsk to ip-10-0-132-81.ec2.internal metadata: creationTimestamp: "2026-04-24T21:30:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:30:03Z" name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk.18a9684c01ba30d3 namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: bd1cac30-0579-43f8-93c6-dacbecd56aab reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:03Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk namespace: kserve-ci-e2e-test resourceVersion: "15362" uid: 319d9352-b194-4a33-83c2-3906eaa110fd kind: Event lastTimestamp: "2026-04-24T21:30:03Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-b868d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:03Z" name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk.18a9684c0ce28254 namespace: kserve-ci-e2e-test resourceVersion: "15380" uid: db66c60b-95c1-453c-bb48-e298c02c3ffa reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:04Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk namespace: kserve-ci-e2e-test resourceVersion: "15371" uid: 319d9352-b194-4a33-83c2-3906eaa110fd kind: Event lastTimestamp: "2026-04-24T21:30:04Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:30:04Z" name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk.18a9684c3ecac305 namespace: kserve-ci-e2e-test resourceVersion: "15394" uid: fcc7ad78-cc0d-46a8-9dd2-97cbb719ff6a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk namespace: kserve-ci-e2e-test resourceVersion: "15362" uid: 319d9352-b194-4a33-83c2-3906eaa110fd kind: Event lastTimestamp: "2026-04-24T21:30:04Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1447" already present on machine metadata: creationTimestamp: "2026-04-24T21:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:04Z" name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk.18a9684c400c03e1 namespace: kserve-ci-e2e-test resourceVersion: "15396" uid: a0fec521-f599-45db-81e4-8e8854a79b5b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk namespace: kserve-ci-e2e-test resourceVersion: "15362" uid: 319d9352-b194-4a33-83c2-3906eaa110fd kind: Event lastTimestamp: "2026-04-24T21:30:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:04Z" name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk.18a9684c43325e3b namespace: kserve-ci-e2e-test resourceVersion: "15399" uid: 0d8d4aaf-121b-49c1-b8d6-461b9c16a639 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk namespace: kserve-ci-e2e-test resourceVersion: "15362" uid: 319d9352-b194-4a33-83c2-3906eaa110fd kind: Event lastTimestamp: "2026-04-24T21:30:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:04Z" name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk.18a9684c43de3bb1 namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: 6ba34c61-2c62-45b2-90d8-d73514913918 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk namespace: kserve-ci-e2e-test resourceVersion: "15362" uid: 319d9352-b194-4a33-83c2-3906eaa110fd kind: Event lastTimestamp: "2026-04-24T21:30:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:04Z" name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk.18a9684c43e9561e namespace: kserve-ci-e2e-test resourceVersion: "15401" uid: d21db9fb-8996-4cac-bcc8-3c2828adc4c0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk namespace: kserve-ci-e2e-test resourceVersion: "15362" uid: 319d9352-b194-4a33-83c2-3906eaa110fd kind: Event lastTimestamp: "2026-04-24T21:30:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:04Z" name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk.18a9684c46877035 namespace: kserve-ci-e2e-test resourceVersion: "15406" uid: 7b447e68-18b5-422d-ac20-a8adc183f1e1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk namespace: kserve-ci-e2e-test resourceVersion: "15362" uid: 319d9352-b194-4a33-83c2-3906eaa110fd kind: Event lastTimestamp: "2026-04-24T21:30:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:04Z" name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk.18a9684c46f859ec namespace: kserve-ci-e2e-test resourceVersion: "15407" uid: 7c80950c-b3d5-4b53-8d95-01085c0edc47 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk namespace: kserve-ci-e2e-test resourceVersion: "15362" uid: 319d9352-b194-4a33-83c2-3906eaa110fd kind: Event lastTimestamp: "2026-04-24T21:30:41Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:41Z" name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk.18a9684c6a751e58 namespace: kserve-ci-e2e-test resourceVersion: "15924" uid: 3b6306d5-f416-430b-8d93-ea7b893adc0e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk namespace: kserve-ci-e2e-test resourceVersion: "15362" uid: 319d9352-b194-4a33-83c2-3906eaa110fd kind: Event lastTimestamp: "2026-04-24T21:39:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:17Z" name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk.18a968cd12306106 namespace: kserve-ci-e2e-test resourceVersion: "19722" uid: d9ee43db-5615-41e1-9f7c-607366ebdb08 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk namespace: kserve-ci-e2e-test resourceVersion: "15362" uid: 319d9352-b194-4a33-83c2-3906eaa110fd kind: Event lastTimestamp: "2026-04-24T21:39:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:17Z" name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk.18a968cd1230fdb2 namespace: kserve-ci-e2e-test resourceVersion: "19726" uid: 214e1800-1dbe-4cf6-a2db-dce650190846 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk namespace: kserve-ci-e2e-test resourceVersion: "15362" uid: 319d9352-b194-4a33-83c2-3906eaa110fd kind: Event lastTimestamp: "2026-04-24T21:39:22Z" message: 'Readiness probe failed: Get "https://10.133.0.29:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-04-24T21:39:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:22Z" name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk.18a968ce1a8e35be namespace: kserve-ci-e2e-test resourceVersion: "19830" uid: d6f948fc-3d60-4ca2-9acf-c269f711df38 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk namespace: kserve-ci-e2e-test resourceVersion: "15362" uid: 319d9352-b194-4a33-83c2-3906eaa110fd kind: Event lastTimestamp: "2026-04-24T21:39:22Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: i/o timeout' metadata: creationTimestamp: "2026-04-24T21:39:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:22Z" name: error-404-isvc-b868d-predictor-5b5b76b898-87zsk.18a968ce1af4f6eb namespace: kserve-ci-e2e-test resourceVersion: "19831" uid: 33b7f076-a556-4d54-9a6b-f8115d352130 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b868d-predictor-5b5b76b898 namespace: kserve-ci-e2e-test resourceVersion: "15354" uid: fa763035-6d27-461b-8d51-27fe963b1aed kind: Event lastTimestamp: "2026-04-24T21:30:03Z" message: 'Created pod: error-404-isvc-b868d-predictor-5b5b76b898-87zsk' metadata: creationTimestamp: "2026-04-24T21:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:03Z" name: error-404-isvc-b868d-predictor-5b5b76b898.18a9684c01515d38 namespace: kserve-ci-e2e-test resourceVersion: "15360" uid: b938b66a-e41f-4d9c-bba5-8ad94343f866 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b868d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15352" uid: 7272b2c0-b56c-4f57-9ecf-03946613e2b0 kind: Event lastTimestamp: "2026-04-24T21:30:03Z" message: Scaled up replica set error-404-isvc-b868d-predictor-5b5b76b898 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:03Z" name: error-404-isvc-b868d-predictor.18a9684bf978b6d9 namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: 85d8f385-791f-4826-96aa-f72d1021d80d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:30:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b868d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15353" uid: 0105129a-b614-411f-bfb6-76f4e4380911 kind: Event lastTimestamp: "2026-04-24T21:30:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:48Z" name: error-404-isvc-b868d-predictor.18a9684f77ec47ae namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: d0ebd176-fc3d-4c99-9575-8a8c20985312 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:30:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b868d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15353" uid: 0105129a-b614-411f-bfb6-76f4e4380911 kind: Event lastTimestamp: "2026-04-24T21:30:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:48Z" name: error-404-isvc-b868d-predictor.18a9684f7848b43c namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 3c5f7264-2cd7-485d-a700-a24c34c8bcdd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:31:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b868d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: 0105129a-b614-411f-bfb6-76f4e4380911 kind: Event lastTimestamp: "2026-04-24T21:31:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:31:33Z" name: error-404-isvc-b868d-predictor.18a96859f4994500 namespace: kserve-ci-e2e-test resourceVersion: "16434" uid: 9978ad0d-4bf6-42ce-aeb4-5858f2fd7659 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:31:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b868d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: 0105129a-b614-411f-bfb6-76f4e4380911 kind: Event lastTimestamp: "2026-04-24T21:31:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:31:33Z" name: error-404-isvc-b868d-predictor.18a96859f5392b0a namespace: kserve-ci-e2e-test resourceVersion: "16435" uid: b9f9061b-6024-4133-9d48-8d5f8118eda0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:30:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b868d namespace: kserve-ci-e2e-test resourceVersion: "15328" uid: b43a0c83-3abe-4517-93dc-78b386f2ce55 kind: Event lastTimestamp: "2026-04-24T21:30:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:30:51Z" name: error-404-isvc-b868d.18a9684bf959ccd4 namespace: kserve-ci-e2e-test resourceVersion: "16041" uid: 41ee69a1-c94f-4dcc-9232-d4b2db0f6248 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b868d namespace: kserve-ci-e2e-test resourceVersion: "15328" uid: b43a0c83-3abe-4517-93dc-78b386f2ce55 kind: Event lastTimestamp: "2026-04-24T21:30:03Z" message: 'Failed to update status for InferenceService "error-404-isvc-b868d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b868d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:30:03Z" name: error-404-isvc-b868d.18a9684bf9b4a449 namespace: kserve-ci-e2e-test resourceVersion: "15370" uid: 3adeb8c4-6204-447a-93ad-70be74b2b0f7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b868d namespace: kserve-ci-e2e-test resourceVersion: "15328" uid: b43a0c83-3abe-4517-93dc-78b386f2ce55 kind: Event lastTimestamp: "2026-04-24T21:30:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b868d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:30:03Z" name: error-404-isvc-b868d.18a9684bf9b4f4b0 namespace: kserve-ci-e2e-test resourceVersion: "15378" uid: 6c54f02f-7bf1-4c09-9406-fc203ca58c42 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b868d namespace: kserve-ci-e2e-test resourceVersion: "16039" uid: b43a0c83-3abe-4517-93dc-78b386f2ce55 kind: Event lastTimestamp: "2026-04-24T21:30:51Z" message: InferenceService [error-404-isvc-b868d] is Ready metadata: creationTimestamp: "2026-04-24T21:30:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:30:51Z" name: error-404-isvc-b868d.18a9685725304e2e namespace: kserve-ci-e2e-test resourceVersion: "16040" uid: 9bb23dba-9dc9-4a46-b0c1-5a60c654df1b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:39:54.028105Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh namespace: kserve-ci-e2e-test resourceVersion: "20171" uid: cdf8dd72-b419-4e7d-9ee4-627089900967 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c1a35-predictor-5b5885dd-sx2hh to ip-10-0-132-81.ec2.internal metadata: creationTimestamp: "2026-04-24T21:39:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:39:54Z" name: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh.18a968d578302153 namespace: kserve-ci-e2e-test resourceVersion: "20175" uid: b3f88397-28a8-4e78-9c82-84a436db5bc8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:54Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh namespace: kserve-ci-e2e-test resourceVersion: "20173" uid: cdf8dd72-b419-4e7d-9ee4-627089900967 kind: Event lastTimestamp: "2026-04-24T21:39:54Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-c1a35-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:39:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:54Z" name: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh.18a968d581a1dbaa namespace: kserve-ci-e2e-test resourceVersion: "20183" uid: ebd3300e-2f4c-4c44-a10e-8c9f0d4be18b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:55Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh namespace: kserve-ci-e2e-test resourceVersion: "20181" uid: cdf8dd72-b419-4e7d-9ee4-627089900967 kind: Event lastTimestamp: "2026-04-24T21:39:55Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:39:55Z" name: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh.18a968d5b4edb9be namespace: kserve-ci-e2e-test resourceVersion: "20202" uid: 11736647-2b67-49b6-99e7-6ace0c4abc89 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh namespace: kserve-ci-e2e-test resourceVersion: "20173" uid: cdf8dd72-b419-4e7d-9ee4-627089900967 kind: Event lastTimestamp: "2026-04-24T21:39:55Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1447" already present on machine metadata: creationTimestamp: "2026-04-24T21:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:55Z" name: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh.18a968d5b68518d8 namespace: kserve-ci-e2e-test resourceVersion: "20206" uid: 66480f99-77a8-4537-8e02-a709811acef5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh namespace: kserve-ci-e2e-test resourceVersion: "20173" uid: cdf8dd72-b419-4e7d-9ee4-627089900967 kind: Event lastTimestamp: "2026-04-24T21:39:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:55Z" name: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh.18a968d5b9943527 namespace: kserve-ci-e2e-test resourceVersion: "20209" uid: 85cef9f5-aefe-4153-a64e-78e23dac80be reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh namespace: kserve-ci-e2e-test resourceVersion: "20173" uid: cdf8dd72-b419-4e7d-9ee4-627089900967 kind: Event lastTimestamp: "2026-04-24T21:39:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:55Z" name: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh.18a968d5ba1cc3bb namespace: kserve-ci-e2e-test resourceVersion: "20210" uid: d7615b2a-b8d6-4fad-a398-c60e9fc3015e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh namespace: kserve-ci-e2e-test resourceVersion: "20173" uid: cdf8dd72-b419-4e7d-9ee4-627089900967 kind: Event lastTimestamp: "2026-04-24T21:39:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:55Z" name: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh.18a968d5ba2a57c4 namespace: kserve-ci-e2e-test resourceVersion: "20212" uid: 7e10e4e6-f0af-49de-bd28-4e0dd309bf19 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh namespace: kserve-ci-e2e-test resourceVersion: "20173" uid: cdf8dd72-b419-4e7d-9ee4-627089900967 kind: Event lastTimestamp: "2026-04-24T21:39:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:55Z" name: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh.18a968d5bce772c7 namespace: kserve-ci-e2e-test resourceVersion: "20213" uid: a628c872-45e0-4d90-8a82-7305145f3cd3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh namespace: kserve-ci-e2e-test resourceVersion: "20173" uid: cdf8dd72-b419-4e7d-9ee4-627089900967 kind: Event lastTimestamp: "2026-04-24T21:39:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:55Z" name: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh.18a968d5bd582d99 namespace: kserve-ci-e2e-test resourceVersion: "20214" uid: 9af98a33-49f6-48d9-a575-e47bb7a529ee reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:39:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh namespace: kserve-ci-e2e-test resourceVersion: "20173" uid: cdf8dd72-b419-4e7d-9ee4-627089900967 kind: Event lastTimestamp: "2026-04-24T21:40:32Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:39:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:32Z" name: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh.18a968d601bf3314 namespace: kserve-ci-e2e-test resourceVersion: "20733" uid: b8a647d9-2e24-4f5b-9540-ab84a5e574f1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh namespace: kserve-ci-e2e-test resourceVersion: "20173" uid: cdf8dd72-b419-4e7d-9ee4-627089900967 kind: Event lastTimestamp: "2026-04-24T21:41:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:04Z" name: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh.18a968e5c5f2330c namespace: kserve-ci-e2e-test resourceVersion: "21081" uid: 519d1182-4860-41ac-abd0-c8b69d113033 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh namespace: kserve-ci-e2e-test resourceVersion: "20173" uid: cdf8dd72-b419-4e7d-9ee4-627089900967 kind: Event lastTimestamp: "2026-04-24T21:41:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:04Z" name: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh.18a968e5c5f2c844 namespace: kserve-ci-e2e-test resourceVersion: "21084" uid: 13c7685e-8eb3-4ccf-b783-e0564b5875c5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh namespace: kserve-ci-e2e-test resourceVersion: "20173" uid: cdf8dd72-b419-4e7d-9ee4-627089900967 kind: Event lastTimestamp: "2026-04-24T21:41:07Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:41:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:07Z" name: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh.18a968e68a021c3f namespace: kserve-ci-e2e-test resourceVersion: "21207" uid: efe3988d-79e5-4e0e-9bc0-3ba0ab30274e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c1a35-predictor-5b5885dd namespace: kserve-ci-e2e-test resourceVersion: "20164" uid: 1e84b4c0-26f8-4ecf-8e6a-8e2b8d0c927d kind: Event lastTimestamp: "2026-04-24T21:39:54Z" message: 'Created pod: error-404-isvc-c1a35-predictor-5b5885dd-sx2hh' metadata: creationTimestamp: "2026-04-24T21:39:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:54Z" name: error-404-isvc-c1a35-predictor-5b5885dd.18a968d577b8cde9 namespace: kserve-ci-e2e-test resourceVersion: "20172" uid: a15c72de-0717-4da6-9241-aacab49fbb3d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c1a35-predictor namespace: kserve-ci-e2e-test resourceVersion: "20161" uid: c61b61f3-2c61-4e31-a0ec-2962c9cba9a1 kind: Event lastTimestamp: "2026-04-24T21:39:53Z" message: Scaled up replica set error-404-isvc-c1a35-predictor-5b5885dd from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:39:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:53Z" name: error-404-isvc-c1a35-predictor.18a968d575e16f23 namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: 65252bc4-38c5-437c-95be-c406a20eb6c9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:40:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c1a35-predictor namespace: kserve-ci-e2e-test resourceVersion: "20163" uid: 63c7ee80-d32f-4efb-a78e-8d93d5b7283c kind: Event lastTimestamp: "2026-04-24T21:40:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:40:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:24Z" name: error-404-isvc-c1a35-predictor.18a968d8f45fe1e5 namespace: kserve-ci-e2e-test resourceVersion: "20523" uid: 40978e42-790b-4fb3-a713-ae8ce3d39f91 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:40:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c1a35-predictor namespace: kserve-ci-e2e-test resourceVersion: "20163" uid: 63c7ee80-d32f-4efb-a78e-8d93d5b7283c kind: Event lastTimestamp: "2026-04-24T21:40:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:40:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:24Z" name: error-404-isvc-c1a35-predictor.18a968d8f4cb42d1 namespace: kserve-ci-e2e-test resourceVersion: "20524" uid: 5667c358-d62a-43bd-9b76-c63555f032d1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:40:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c1a35-predictor namespace: kserve-ci-e2e-test resourceVersion: "20365" uid: 63c7ee80-d32f-4efb-a78e-8d93d5b7283c kind: Event lastTimestamp: "2026-04-24T21:40:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:40:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:54Z" name: error-404-isvc-c1a35-predictor.18a968dff260b2c1 namespace: kserve-ci-e2e-test resourceVersion: "20950" uid: 224ffb33-383b-45bb-8d73-1447537e26e4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:40:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c1a35-predictor namespace: kserve-ci-e2e-test resourceVersion: "20365" uid: 63c7ee80-d32f-4efb-a78e-8d93d5b7283c kind: Event lastTimestamp: "2026-04-24T21:40:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:40:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:54Z" name: error-404-isvc-c1a35-predictor.18a968dff2a3ade0 namespace: kserve-ci-e2e-test resourceVersion: "20951" uid: df1935e7-0409-4ab1-949e-2149d309cc67 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:39:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c1a35 namespace: kserve-ci-e2e-test resourceVersion: "20132" uid: 02020633-b4b0-4c8f-83ce-753d8034cd7b kind: Event lastTimestamp: "2026-04-24T21:40:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:39:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:40:42Z" name: error-404-isvc-c1a35.18a968d575bbd1b4 namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: 73f0538d-fd94-4ef0-a487-9d2ce5112d66 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c1a35 namespace: kserve-ci-e2e-test resourceVersion: "20819" uid: 02020633-b4b0-4c8f-83ce-753d8034cd7b kind: Event lastTimestamp: "2026-04-24T21:40:42Z" message: InferenceService [error-404-isvc-c1a35] is Ready metadata: creationTimestamp: "2026-04-24T21:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:40:42Z" name: error-404-isvc-c1a35.18a968e0bcc4c9cd namespace: kserve-ci-e2e-test resourceVersion: "20820" uid: 7d0cdacd-6331-4f18-9ee9-4648f7935f14 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:28:29.036577Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d41b1-predictor-597b847c78-mknpz namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: 404559c5-ccb0-43a1-80ce-1c7609bc5692 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d41b1-predictor-597b847c78-mknpz to ip-10-0-132-81.ec2.internal metadata: creationTimestamp: "2026-04-24T21:28:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:28:29Z" name: error-404-isvc-d41b1-predictor-597b847c78-mknpz.18a96835fb82ecf7 namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: e324e059-150a-40df-87cf-7d29511b01df reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:29Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d41b1-predictor-597b847c78-mknpz namespace: kserve-ci-e2e-test resourceVersion: "14240" uid: 404559c5-ccb0-43a1-80ce-1c7609bc5692 kind: Event lastTimestamp: "2026-04-24T21:28:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-d41b1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:29Z" name: error-404-isvc-d41b1-predictor-597b847c78-mknpz.18a968360762bea3 namespace: kserve-ci-e2e-test resourceVersion: "14272" uid: 657ae1a0-b32c-4773-b871-7b38d4ef1b1a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:30Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d41b1-predictor-597b847c78-mknpz namespace: kserve-ci-e2e-test resourceVersion: "14249" uid: 404559c5-ccb0-43a1-80ce-1c7609bc5692 kind: Event lastTimestamp: "2026-04-24T21:28:30Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:28:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:28:30Z" name: error-404-isvc-d41b1-predictor-597b847c78-mknpz.18a9683639e80aa1 namespace: kserve-ci-e2e-test resourceVersion: "14322" uid: f4f8958f-3efa-490c-b9c7-cdd703110c10 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d41b1-predictor-597b847c78-mknpz namespace: kserve-ci-e2e-test resourceVersion: "14240" uid: 404559c5-ccb0-43a1-80ce-1c7609bc5692 kind: Event lastTimestamp: "2026-04-24T21:28:30Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1447" metadata: creationTimestamp: "2026-04-24T21:28:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:30Z" name: error-404-isvc-d41b1-predictor-597b847c78-mknpz.18a968363b2d186d namespace: kserve-ci-e2e-test resourceVersion: "14325" uid: ca40f1dd-df26-4726-95d9-f7c83d8456bf reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d41b1-predictor-597b847c78-mknpz namespace: kserve-ci-e2e-test resourceVersion: "14240" uid: 404559c5-ccb0-43a1-80ce-1c7609bc5692 kind: Event lastTimestamp: "2026-04-24T21:28:44Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1447" in 14.238s (14.238s including waiting). Image size: 1334715495 bytes.' metadata: creationTimestamp: "2026-04-24T21:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:44Z" name: error-404-isvc-d41b1-predictor-597b847c78-mknpz.18a968398bd7c996 namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: e2ad72ff-e0d3-4550-87a1-1365f5b64c27 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d41b1-predictor-597b847c78-mknpz namespace: kserve-ci-e2e-test resourceVersion: "14240" uid: 404559c5-ccb0-43a1-80ce-1c7609bc5692 kind: Event lastTimestamp: "2026-04-24T21:28:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:44Z" name: error-404-isvc-d41b1-predictor-597b847c78-mknpz.18a968398f40c937 namespace: kserve-ci-e2e-test resourceVersion: "14481" uid: f10e641a-5a97-46a5-ae02-3aaa8691c36f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d41b1-predictor-597b847c78-mknpz namespace: kserve-ci-e2e-test resourceVersion: "14240" uid: 404559c5-ccb0-43a1-80ce-1c7609bc5692 kind: Event lastTimestamp: "2026-04-24T21:28:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:44Z" name: error-404-isvc-d41b1-predictor-597b847c78-mknpz.18a968398fd8a30e namespace: kserve-ci-e2e-test resourceVersion: "14482" uid: 76f460eb-7780-4fc9-9daa-b1f2ea74ed7d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d41b1-predictor-597b847c78-mknpz namespace: kserve-ci-e2e-test resourceVersion: "14240" uid: 404559c5-ccb0-43a1-80ce-1c7609bc5692 kind: Event lastTimestamp: "2026-04-24T21:28:44Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T21:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:44Z" name: error-404-isvc-d41b1-predictor-597b847c78-mknpz.18a968398fe76af1 namespace: kserve-ci-e2e-test resourceVersion: "14483" uid: 78f4a84b-6d7d-4008-9983-882f7763631d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d41b1-predictor-597b847c78-mknpz namespace: kserve-ci-e2e-test resourceVersion: "14240" uid: 404559c5-ccb0-43a1-80ce-1c7609bc5692 kind: Event lastTimestamp: "2026-04-24T21:28:46Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.576s (2.576s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T21:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:46Z" name: error-404-isvc-d41b1-predictor-597b847c78-mknpz.18a9683a29754055 namespace: kserve-ci-e2e-test resourceVersion: "14532" uid: 0b3c453b-db2e-47a6-ad18-6906728408bf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d41b1-predictor-597b847c78-mknpz namespace: kserve-ci-e2e-test resourceVersion: "14240" uid: 404559c5-ccb0-43a1-80ce-1c7609bc5692 kind: Event lastTimestamp: "2026-04-24T21:28:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:47Z" name: error-404-isvc-d41b1-predictor-597b847c78-mknpz.18a9683a2cafc8fd namespace: kserve-ci-e2e-test resourceVersion: "14534" uid: 6c030a93-50c4-4658-974d-af7d59f48ba4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d41b1-predictor-597b847c78-mknpz namespace: kserve-ci-e2e-test resourceVersion: "14240" uid: 404559c5-ccb0-43a1-80ce-1c7609bc5692 kind: Event lastTimestamp: "2026-04-24T21:28:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:47Z" name: error-404-isvc-d41b1-predictor-597b847c78-mknpz.18a9683a2d27288b namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: d883f6d4-d816-4ccb-b8bb-574e5cecd64b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:28:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d41b1-predictor-597b847c78-mknpz namespace: kserve-ci-e2e-test resourceVersion: "14240" uid: 404559c5-ccb0-43a1-80ce-1c7609bc5692 kind: Event lastTimestamp: "2026-04-24T21:30:03Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:03Z" name: error-404-isvc-d41b1-predictor-597b847c78-mknpz.18a9683a3139674c namespace: kserve-ci-e2e-test resourceVersion: "15332" uid: cd4734da-5c9b-4404-91db-f574ebe20f9f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d41b1-predictor-597b847c78-mknpz namespace: kserve-ci-e2e-test resourceVersion: "14240" uid: 404559c5-ccb0-43a1-80ce-1c7609bc5692 kind: Event lastTimestamp: "2026-04-24T21:30:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:03Z" name: error-404-isvc-d41b1-predictor-597b847c78-mknpz.18a9684bdf93ef9b namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: a2daa7dd-bfbd-4990-a86d-115733b905a5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d41b1-predictor-597b847c78-mknpz namespace: kserve-ci-e2e-test resourceVersion: "14240" uid: 404559c5-ccb0-43a1-80ce-1c7609bc5692 kind: Event lastTimestamp: "2026-04-24T21:30:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:03Z" name: error-404-isvc-d41b1-predictor-597b847c78-mknpz.18a9684bdf949c36 namespace: kserve-ci-e2e-test resourceVersion: "15327" uid: 3693aaa6-3790-48cb-bd49-03075b04fd05 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d41b1-predictor-597b847c78-mknpz namespace: kserve-ci-e2e-test resourceVersion: "14240" uid: 404559c5-ccb0-43a1-80ce-1c7609bc5692 kind: Event lastTimestamp: "2026-04-24T21:30:03Z" message: 'Readiness probe failed: Get "https://10.133.0.27:8643/healthz": dial tcp 10.133.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:03Z" name: error-404-isvc-d41b1-predictor-597b847c78-mknpz.18a9684be365e3e1 namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: 0389be79-3115-4def-b8b3-9d9703ade697 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d41b1-predictor-597b847c78 namespace: kserve-ci-e2e-test resourceVersion: "14224" uid: 5efe9787-8dbb-4b8f-b687-c7a50b9522fd kind: Event lastTimestamp: "2026-04-24T21:28:29Z" message: 'Created pod: error-404-isvc-d41b1-predictor-597b847c78-mknpz' metadata: creationTimestamp: "2026-04-24T21:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:28:29Z" name: error-404-isvc-d41b1-predictor-597b847c78.18a96835fb0dfc35 namespace: kserve-ci-e2e-test resourceVersion: "14238" uid: 165259e8-117d-468a-b8fb-e7d473f05b6e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d41b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 68bb277a-ec7a-462d-9e5e-f1fc5ca86a4d kind: Event lastTimestamp: "2026-04-24T21:28:29Z" message: Scaled up replica set error-404-isvc-d41b1-predictor-597b847c78 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:28:29Z" name: error-404-isvc-d41b1-predictor.18a96835f97189ea namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: a4c9f6bf-2de1-41c7-b5ee-7c7f78ba0a32 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:28:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d41b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14223" uid: 9694c24d-0f61-4dba-a6a4-e115cb8b4302 kind: Event lastTimestamp: "2026-04-24T21:29:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:29:14Z" name: error-404-isvc-d41b1-predictor.18a9683977d0bf9e namespace: kserve-ci-e2e-test resourceVersion: "14794" uid: a661a251-3ff9-42d5-bacd-cf996f601794 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:28:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d41b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14223" uid: 9694c24d-0f61-4dba-a6a4-e115cb8b4302 kind: Event lastTimestamp: "2026-04-24T21:29:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:29:14Z" name: error-404-isvc-d41b1-predictor.18a9683978395a45 namespace: kserve-ci-e2e-test resourceVersion: "14795" uid: bcb4ba05-38b4-4bc4-b331-7bc06ce0238b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:29:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d41b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14469" uid: 9694c24d-0f61-4dba-a6a4-e115cb8b4302 kind: Event lastTimestamp: "2026-04-24T21:29:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:29:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:29:59Z" name: error-404-isvc-d41b1-predictor.18a96843f482b13c namespace: kserve-ci-e2e-test resourceVersion: "15207" uid: 24d1072d-af9c-4c02-9004-5fa9863b14a6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:29:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d41b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14469" uid: 9694c24d-0f61-4dba-a6a4-e115cb8b4302 kind: Event lastTimestamp: "2026-04-24T21:29:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:29:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:29:59Z" name: error-404-isvc-d41b1-predictor.18a96843f4e856ae namespace: kserve-ci-e2e-test resourceVersion: "15208" uid: 7792526a-a4d4-4ca0-9deb-6ae0156cb7c6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:28:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d41b1 namespace: kserve-ci-e2e-test resourceVersion: "14215" uid: 3cf8af79-5836-431b-b3fc-0787b871bec2 kind: Event lastTimestamp: "2026-04-24T21:29:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:28:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:29:33Z" name: error-404-isvc-d41b1.18a96835f93c08b4 namespace: kserve-ci-e2e-test resourceVersion: "14965" uid: 9aa4e67b-0597-40f7-81df-490fdd90d6bf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d41b1 namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: 3cf8af79-5836-431b-b3fc-0787b871bec2 kind: Event lastTimestamp: "2026-04-24T21:29:33Z" message: InferenceService [error-404-isvc-d41b1] is Ready metadata: creationTimestamp: "2026-04-24T21:29:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:29:33Z" name: error-404-isvc-d41b1.18a96844ec00bd67 namespace: kserve-ci-e2e-test resourceVersion: "14964" uid: 3b0f50ed-c92e-43c8-be8f-1948a17592f9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:39:18.042487Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f3231-predictor-69f59b6d96-7wdnc namespace: kserve-ci-e2e-test resourceVersion: "19748" uid: c3ff008d-cbf1-453b-b1a9-541247ef77fd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f3231-predictor-69f59b6d96-7wdnc to ip-10-0-132-81.ec2.internal metadata: creationTimestamp: "2026-04-24T21:39:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:39:18Z" name: error-404-isvc-f3231-predictor-69f59b6d96-7wdnc.18a968cd1747289b namespace: kserve-ci-e2e-test resourceVersion: "19752" uid: 12c7626e-e567-42aa-8592-359f279bba0d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:18Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f3231-predictor-69f59b6d96-7wdnc namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: c3ff008d-cbf1-453b-b1a9-541247ef77fd kind: Event lastTimestamp: "2026-04-24T21:39:18Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-f3231-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:18Z" name: error-404-isvc-f3231-predictor-69f59b6d96-7wdnc.18a968cd2308ceca namespace: kserve-ci-e2e-test resourceVersion: "19761" uid: c87d6a73-fd0a-4ad4-a451-43331f86796c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:19Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f3231-predictor-69f59b6d96-7wdnc namespace: kserve-ci-e2e-test resourceVersion: "19759" uid: c3ff008d-cbf1-453b-b1a9-541247ef77fd kind: Event lastTimestamp: "2026-04-24T21:39:19Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:39:19Z" name: error-404-isvc-f3231-predictor-69f59b6d96-7wdnc.18a968cd54a6c0d1 namespace: kserve-ci-e2e-test resourceVersion: "19782" uid: 1b5a7526-318a-46b0-80ee-3eef740a1949 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f3231-predictor-69f59b6d96-7wdnc namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: c3ff008d-cbf1-453b-b1a9-541247ef77fd kind: Event lastTimestamp: "2026-04-24T21:39:19Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1447" already present on machine metadata: creationTimestamp: "2026-04-24T21:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:19Z" name: error-404-isvc-f3231-predictor-69f59b6d96-7wdnc.18a968cd5733eefc namespace: kserve-ci-e2e-test resourceVersion: "19784" uid: 4ab062ef-85c0-4099-8b87-a648c9b86699 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f3231-predictor-69f59b6d96-7wdnc namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: c3ff008d-cbf1-453b-b1a9-541247ef77fd kind: Event lastTimestamp: "2026-04-24T21:39:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:19Z" name: error-404-isvc-f3231-predictor-69f59b6d96-7wdnc.18a968cd5ab0820a namespace: kserve-ci-e2e-test resourceVersion: "19785" uid: c28e18cc-7ad0-47e9-ac04-d775d2daf854 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f3231-predictor-69f59b6d96-7wdnc namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: c3ff008d-cbf1-453b-b1a9-541247ef77fd kind: Event lastTimestamp: "2026-04-24T21:39:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:19Z" name: error-404-isvc-f3231-predictor-69f59b6d96-7wdnc.18a968cd5b27eda2 namespace: kserve-ci-e2e-test resourceVersion: "19786" uid: 3a80e6b2-0c0e-4399-809a-1be273ec0d39 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f3231-predictor-69f59b6d96-7wdnc namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: c3ff008d-cbf1-453b-b1a9-541247ef77fd kind: Event lastTimestamp: "2026-04-24T21:39:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:19Z" name: error-404-isvc-f3231-predictor-69f59b6d96-7wdnc.18a968cd5b36251e namespace: kserve-ci-e2e-test resourceVersion: "19787" uid: 08fcc5b4-ee90-408d-9980-622fef1c540c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f3231-predictor-69f59b6d96-7wdnc namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: c3ff008d-cbf1-453b-b1a9-541247ef77fd kind: Event lastTimestamp: "2026-04-24T21:39:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:19Z" name: error-404-isvc-f3231-predictor-69f59b6d96-7wdnc.18a968cd5e471ae5 namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: f4a461dc-c78e-4303-8874-f50545387fff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f3231-predictor-69f59b6d96-7wdnc namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: c3ff008d-cbf1-453b-b1a9-541247ef77fd kind: Event lastTimestamp: "2026-04-24T21:39:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:19Z" name: error-404-isvc-f3231-predictor-69f59b6d96-7wdnc.18a968cd5ec060f5 namespace: kserve-ci-e2e-test resourceVersion: "19790" uid: c3170ba6-30cf-43b3-a38e-b8b1f6ca09dc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:39:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f3231-predictor-69f59b6d96-7wdnc namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: c3ff008d-cbf1-453b-b1a9-541247ef77fd kind: Event lastTimestamp: "2026-04-24T21:39:57Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:39:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:57Z" name: error-404-isvc-f3231-predictor-69f59b6d96-7wdnc.18a968cdd485740d namespace: kserve-ci-e2e-test resourceVersion: "20243" uid: 93404d64-d297-4f32-ad8d-ac612d53e2fb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f3231-predictor-69f59b6d96-7wdnc namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: c3ff008d-cbf1-453b-b1a9-541247ef77fd kind: Event lastTimestamp: "2026-04-24T21:40:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:28Z" name: error-404-isvc-f3231-predictor-69f59b6d96-7wdnc.18a968dd70363cd6 namespace: kserve-ci-e2e-test resourceVersion: "20641" uid: 7f90d33c-dce9-4aea-9cad-d0ef2e6836b4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f3231-predictor-69f59b6d96-7wdnc namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: c3ff008d-cbf1-453b-b1a9-541247ef77fd kind: Event lastTimestamp: "2026-04-24T21:40:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:28Z" name: error-404-isvc-f3231-predictor-69f59b6d96-7wdnc.18a968dd70389a34 namespace: kserve-ci-e2e-test resourceVersion: "20643" uid: 2684fc84-13f0-4cb0-b7e5-bfc09f8ea15b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f3231-predictor-69f59b6d96 namespace: kserve-ci-e2e-test resourceVersion: "19740" uid: f03521de-423a-432b-af40-aea111fac6ed kind: Event lastTimestamp: "2026-04-24T21:39:18Z" message: 'Created pod: error-404-isvc-f3231-predictor-69f59b6d96-7wdnc' metadata: creationTimestamp: "2026-04-24T21:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:18Z" name: error-404-isvc-f3231-predictor-69f59b6d96.18a968cd16a685de namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: 8f7b32ac-d41c-47b1-87b4-6579b7e2cb8b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f3231-predictor namespace: kserve-ci-e2e-test resourceVersion: "19738" uid: f0a0b77c-faa0-40ba-a7b7-2eb1e5c2822d kind: Event lastTimestamp: "2026-04-24T21:39:18Z" message: Scaled up replica set error-404-isvc-f3231-predictor-69f59b6d96 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:18Z" name: error-404-isvc-f3231-predictor.18a968cd14f9addd namespace: kserve-ci-e2e-test resourceVersion: "19743" uid: 08c0c148-3da8-49b1-8e84-1136789fbe88 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:39:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f3231-predictor namespace: kserve-ci-e2e-test resourceVersion: "19739" uid: 1ffe6f66-d90b-4d8f-bddc-3bef1549d8cd kind: Event lastTimestamp: "2026-04-24T21:40:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:39:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:03Z" name: error-404-isvc-f3231-predictor.18a968d093bae543 namespace: kserve-ci-e2e-test resourceVersion: "20301" uid: 1827afcb-e3bf-4d61-abc8-76589dddf2b3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:39:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f3231-predictor namespace: kserve-ci-e2e-test resourceVersion: "19739" uid: 1ffe6f66-d90b-4d8f-bddc-3bef1549d8cd kind: Event lastTimestamp: "2026-04-24T21:40:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:39:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:03Z" name: error-404-isvc-f3231-predictor.18a968d09472223e namespace: kserve-ci-e2e-test resourceVersion: "20302" uid: 15f92bee-641d-4310-af48-9c6923b4337f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:39:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f3231-predictor namespace: kserve-ci-e2e-test resourceVersion: "19919" uid: 1ffe6f66-d90b-4d8f-bddc-3bef1549d8cd kind: Event lastTimestamp: "2026-04-24T21:40:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:18Z" name: error-404-isvc-f3231-predictor.18a968d4131dbf93 namespace: kserve-ci-e2e-test resourceVersion: "20463" uid: eeef8361-69b6-4ca3-a2dd-6938d6addd84 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:39:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f3231-predictor namespace: kserve-ci-e2e-test resourceVersion: "19919" uid: 1ffe6f66-d90b-4d8f-bddc-3bef1549d8cd kind: Event lastTimestamp: "2026-04-24T21:40:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:18Z" name: error-404-isvc-f3231-predictor.18a968d413897760 namespace: kserve-ci-e2e-test resourceVersion: "20464" uid: 953116e7-f3c7-4d91-b191-526fe620e536 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:39:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f3231 namespace: kserve-ci-e2e-test resourceVersion: "19711" uid: f723ec6d-4da8-4e89-8138-342cbd29baaa kind: Event lastTimestamp: "2026-04-24T21:40:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:40:07Z" name: error-404-isvc-f3231.18a968cd14d3afd6 namespace: kserve-ci-e2e-test resourceVersion: "20349" uid: 9d0833a3-a2e5-4263-8e83-a74882567e3b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f3231 namespace: kserve-ci-e2e-test resourceVersion: "19711" uid: f723ec6d-4da8-4e89-8138-342cbd29baaa kind: Event lastTimestamp: "2026-04-24T21:39:18Z" message: 'Failed to update status for InferenceService "error-404-isvc-f3231": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f3231": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:39:18Z" name: error-404-isvc-f3231.18a968cd155db829 namespace: kserve-ci-e2e-test resourceVersion: "19745" uid: ed8e5955-6ed5-46cc-91b8-5c60b1fef8a3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f3231 namespace: kserve-ci-e2e-test resourceVersion: "19711" uid: f723ec6d-4da8-4e89-8138-342cbd29baaa kind: Event lastTimestamp: "2026-04-24T21:39:18Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f3231": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:39:18Z" name: error-404-isvc-f3231.18a968cd155e05bd namespace: kserve-ci-e2e-test resourceVersion: "19746" uid: cff1e30f-66c9-4b35-8d96-1a35a37a69e3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f3231 namespace: kserve-ci-e2e-test resourceVersion: "20347" uid: f723ec6d-4da8-4e89-8138-342cbd29baaa kind: Event lastTimestamp: "2026-04-24T21:40:07Z" message: InferenceService [error-404-isvc-f3231] is Ready metadata: creationTimestamp: "2026-04-24T21:40:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:40:07Z" name: error-404-isvc-f3231.18a968d8902f3e3f namespace: kserve-ci-e2e-test resourceVersion: "20348" uid: 6365dbfc-a9e9-4d02-acba-c993839892a6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:28:29.023372Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch namespace: kserve-ci-e2e-test resourceVersion: "14229" uid: 412b7f5f-3972-48a5-a5b2-ef53db782271 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-66d58bff49-kwvch to ip-10-0-132-81.ec2.internal metadata: creationTimestamp: "2026-04-24T21:28:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:28:29Z" name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch.18a96835fab9566b namespace: kserve-ci-e2e-test resourceVersion: "14235" uid: c1c0353e-adaa-47ff-9089-f9532f06bb9a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:29Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: 412b7f5f-3972-48a5-a5b2-ef53db782271 kind: Event lastTimestamp: "2026-04-24T21:28:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:29Z" name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch.18a968360152be48 namespace: kserve-ci-e2e-test resourceVersion: "14265" uid: f5bcb11e-c045-458a-9943-cef47c2ef629 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:30Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: 412b7f5f-3972-48a5-a5b2-ef53db782271 kind: Event lastTimestamp: "2026-04-24T21:28:30Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:28:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:28:30Z" name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch.18a96836386d6a5d namespace: kserve-ci-e2e-test resourceVersion: "14320" uid: 4872d508-9c0b-4820-be29-3286d5f90c8a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: 412b7f5f-3972-48a5-a5b2-ef53db782271 kind: Event lastTimestamp: "2026-04-24T21:28:30Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:85f85c16414082de64de782f602228652c10c5bace08b518e803a7e911c62e5e" metadata: creationTimestamp: "2026-04-24T21:28:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:30Z" name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch.18a9683639e952ca namespace: kserve-ci-e2e-test resourceVersion: "14323" uid: 79f479c5-6206-435f-9ff8-83d3d59a303a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: 412b7f5f-3972-48a5-a5b2-ef53db782271 kind: Event lastTimestamp: "2026-04-24T21:28:42Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:85f85c16414082de64de782f602228652c10c5bace08b518e803a7e911c62e5e" in 12.593s (12.593s including waiting). Image size: 299849144 bytes.' metadata: creationTimestamp: "2026-04-24T21:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:42Z" name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch.18a96839288a8569 namespace: kserve-ci-e2e-test resourceVersion: "14452" uid: a11b132b-8467-4859-b913-c3b78c73cd15 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: 412b7f5f-3972-48a5-a5b2-ef53db782271 kind: Event lastTimestamp: "2026-04-24T21:28:44Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:44Z" name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch.18a96839890965e2 namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: c07ae028-97af-4c41-a791-788ecd26ba79 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: 412b7f5f-3972-48a5-a5b2-ef53db782271 kind: Event lastTimestamp: "2026-04-24T21:28:44Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:44Z" name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch.18a9683989d3fb46 namespace: kserve-ci-e2e-test resourceVersion: "14479" uid: 2ae49956-c9ce-4f05-bb7b-662fa4fe2009 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: 412b7f5f-3972-48a5-a5b2-ef53db782271 kind: Event lastTimestamp: "2026-04-24T21:28:49Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1447" metadata: creationTimestamp: "2026-04-24T21:28:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:49Z" name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch.18a9683aa8e7522f namespace: kserve-ci-e2e-test resourceVersion: "14557" uid: 65c062d6-03db-4c20-8f6d-72469c36d05e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: 412b7f5f-3972-48a5-a5b2-ef53db782271 kind: Event lastTimestamp: "2026-04-24T21:28:55Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1447" in 6.415s (6.415s including waiting). Image size: 1560926130 bytes.' metadata: creationTimestamp: "2026-04-24T21:28:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:55Z" name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch.18a9683c274fd110 namespace: kserve-ci-e2e-test resourceVersion: "14612" uid: 57c32d36-44cb-4fe1-8502-b91cec0f2a10 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: 412b7f5f-3972-48a5-a5b2-ef53db782271 kind: Event lastTimestamp: "2026-04-24T21:28:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:28:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:55Z" name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch.18a9683c2bd57f24 namespace: kserve-ci-e2e-test resourceVersion: "14614" uid: bc4fca32-4dfd-4866-9024-4c89938fff82 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: 412b7f5f-3972-48a5-a5b2-ef53db782271 kind: Event lastTimestamp: "2026-04-24T21:28:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:28:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:55Z" name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch.18a9683c2c475990 namespace: kserve-ci-e2e-test resourceVersion: "14617" uid: ea33ed20-10df-435d-839d-e44f80feb08d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: 412b7f5f-3972-48a5-a5b2-ef53db782271 kind: Event lastTimestamp: "2026-04-24T21:28:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:28:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:55Z" name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch.18a9683c2c50e652 namespace: kserve-ci-e2e-test resourceVersion: "14618" uid: 16d442db-d2c0-4918-aebe-248b148495b4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: 412b7f5f-3972-48a5-a5b2-ef53db782271 kind: Event lastTimestamp: "2026-04-24T21:28:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:28:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:55Z" name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch.18a9683c2f9d22dd namespace: kserve-ci-e2e-test resourceVersion: "14622" uid: 9330ba20-ea45-4cc7-8c27-8b606acc1cfe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: 412b7f5f-3972-48a5-a5b2-ef53db782271 kind: Event lastTimestamp: "2026-04-24T21:28:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:28:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:55Z" name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch.18a9683c301df447 namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: b77d8d85-2e7a-4a0c-b6cf-7f82136bd328 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:28:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: 412b7f5f-3972-48a5-a5b2-ef53db782271 kind: Event lastTimestamp: "2026-04-24T21:30:42Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:28:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:42Z" name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch.18a9683c4be5c1d1 namespace: kserve-ci-e2e-test resourceVersion: "15937" uid: f668efd3-c2ea-473a-9139-a0da6b8af9c7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: 412b7f5f-3972-48a5-a5b2-ef53db782271 kind: Event lastTimestamp: "2026-04-24T21:30:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:30:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:39Z" name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch.18a968543f1436f6 namespace: kserve-ci-e2e-test resourceVersion: "15842" uid: 1efb6431-a7f1-4ea7-bdba-db68c1c053c1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: 412b7f5f-3972-48a5-a5b2-ef53db782271 kind: Event lastTimestamp: "2026-04-24T21:30:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:30:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:39Z" name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch.18a968543f14cb6a namespace: kserve-ci-e2e-test resourceVersion: "15844" uid: 0c8c2efa-d411-4819-9160-aeff166f2fdc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: 412b7f5f-3972-48a5-a5b2-ef53db782271 kind: Event lastTimestamp: "2026-04-24T21:30:42Z" message: 'Readiness probe failed: Get "https://10.133.0.26:8643/healthz": dial tcp 10.133.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:42Z" name: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch.18a96854fa291648 namespace: kserve-ci-e2e-test resourceVersion: "15935" uid: 3b67de3b-fbce-4aff-bd9f-54bd450661d3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-66d58bff49 namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 6d2d9e87-d08e-48c7-ad0e-cb3fa1e8d3c6 kind: Event lastTimestamp: "2026-04-24T21:28:29Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-66d58bff49-kwvch' metadata: creationTimestamp: "2026-04-24T21:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:28:29Z" name: isvc-sklearn-graph-1-predictor-66d58bff49.18a96835fa4f4056 namespace: kserve-ci-e2e-test resourceVersion: "14230" uid: 711037fb-abb0-44ee-a11a-cbab4f838557 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14207" uid: 0796456d-ad8e-442e-af5f-3e463093f49d kind: Event lastTimestamp: "2026-04-24T21:28:28Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-66d58bff49 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:28:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:28:28Z" name: isvc-sklearn-graph-1-predictor.18a96835f4c59929 namespace: kserve-ci-e2e-test resourceVersion: "14211" uid: fe881194-2ac3-4a68-bedc-ac2dde967ad7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:28:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: 5f44198b-5f0e-4d15-ab26-ab962de8a563 kind: Event lastTimestamp: "2026-04-24T21:29:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:28:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:29:28Z" name: isvc-sklearn-graph-1-predictor.18a9683973386afa namespace: kserve-ci-e2e-test resourceVersion: "14908" uid: c8efe8e1-60a4-433b-8c32-b87080dfca6b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:28:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: 5f44198b-5f0e-4d15-ab26-ab962de8a563 kind: Event lastTimestamp: "2026-04-24T21:29:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:28:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:29:28Z" name: isvc-sklearn-graph-1-predictor.18a9683973802017 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: 79b58105-d435-4360-bdf7-39e8b8761755 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:29:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14466" uid: 5f44198b-5f0e-4d15-ab26-ab962de8a563 kind: Event lastTimestamp: "2026-04-24T21:30:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:29:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:29Z" name: isvc-sklearn-graph-1-predictor.18a968476e983573 namespace: kserve-ci-e2e-test resourceVersion: "15650" uid: da8a732d-b182-451a-8476-bb9b228e06f7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:29:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14466" uid: 5f44198b-5f0e-4d15-ab26-ab962de8a563 kind: Event lastTimestamp: "2026-04-24T21:30:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:29:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:29Z" name: isvc-sklearn-graph-1-predictor.18a968476f01c210 namespace: kserve-ci-e2e-test resourceVersion: "15651" uid: 3d7ac946-3f95-4d97-a5fa-8cdd6a4a6bf6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T21:28:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: b4c9ca2a-0ecb-412a-b0bb-1586a1f93643 kind: Event lastTimestamp: "2026-04-24T21:30:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:28:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:30:02Z" name: isvc-sklearn-graph-1.18a96835f49b9e3f namespace: kserve-ci-e2e-test resourceVersion: "15254" uid: 938f553e-f9f1-412a-afa5-d57e37137b42 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: b4c9ca2a-0ecb-412a-b0bb-1586a1f93643 kind: Event lastTimestamp: "2026-04-24T21:30:02Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-04-24T21:30:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:30:02Z" name: isvc-sklearn-graph-1.18a9684bb119f3ec namespace: kserve-ci-e2e-test resourceVersion: "15252" uid: ba5a9e2f-e2f1-444f-a0c3-bbc0acd2c009 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:28:29.988975Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml namespace: kserve-ci-e2e-test resourceVersion: "14310" uid: facf56c6-f309-4e5d-8667-c2dd2f6fe3f1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-87d6c5875-x4cml to ip-10-0-132-81.ec2.internal metadata: creationTimestamp: "2026-04-24T21:28:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:28:29Z" name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml.18a96836344753b5 namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: cbcb9019-dfa2-4e10-9e09-bb921a87aa5d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:30Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml namespace: kserve-ci-e2e-test resourceVersion: "14318" uid: facf56c6-f309-4e5d-8667-c2dd2f6fe3f1 kind: Event lastTimestamp: "2026-04-24T21:28:30Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:28:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:28:30Z" name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml.18a968364e5a3a6c namespace: kserve-ci-e2e-test resourceVersion: "14329" uid: 80ce07c1-bfa5-44f5-a677-a8be10689149 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml namespace: kserve-ci-e2e-test resourceVersion: "14312" uid: facf56c6-f309-4e5d-8667-c2dd2f6fe3f1 kind: Event lastTimestamp: "2026-04-24T21:28:30Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:85f85c16414082de64de782f602228652c10c5bace08b518e803a7e911c62e5e" metadata: creationTimestamp: "2026-04-24T21:28:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:30Z" name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml.18a968364fab630d namespace: kserve-ci-e2e-test resourceVersion: "14331" uid: ea77923f-02b3-4c36-94bc-0bfea62f09ba reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml namespace: kserve-ci-e2e-test resourceVersion: "14312" uid: facf56c6-f309-4e5d-8667-c2dd2f6fe3f1 kind: Event lastTimestamp: "2026-04-24T21:28:44Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:85f85c16414082de64de782f602228652c10c5bace08b518e803a7e911c62e5e" in 13.784s (13.784s including waiting). Image size: 299849144 bytes.' metadata: creationTimestamp: "2026-04-24T21:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:44Z" name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml.18a96839854ce658 namespace: kserve-ci-e2e-test resourceVersion: "14475" uid: 3de29fea-2b2f-4e66-a892-ee350902bcbb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml namespace: kserve-ci-e2e-test resourceVersion: "14312" uid: facf56c6-f309-4e5d-8667-c2dd2f6fe3f1 kind: Event lastTimestamp: "2026-04-24T21:28:44Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:44Z" name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml.18a9683988fb447b namespace: kserve-ci-e2e-test resourceVersion: "14476" uid: 78c7099b-8a5a-4979-a884-b1ecade19fe1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml namespace: kserve-ci-e2e-test resourceVersion: "14312" uid: facf56c6-f309-4e5d-8667-c2dd2f6fe3f1 kind: Event lastTimestamp: "2026-04-24T21:28:44Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:44Z" name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml.18a9683989a7177a namespace: kserve-ci-e2e-test resourceVersion: "14478" uid: 2324e035-9544-4324-bee5-9eec0596e10b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml namespace: kserve-ci-e2e-test resourceVersion: "14312" uid: facf56c6-f309-4e5d-8667-c2dd2f6fe3f1 kind: Event lastTimestamp: "2026-04-24T21:28:49Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1447" metadata: creationTimestamp: "2026-04-24T21:28:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:49Z" name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml.18a9683aa8faae08 namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: 3fe124d6-20ed-4c00-aa2d-980bbc3d9393 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml namespace: kserve-ci-e2e-test resourceVersion: "14312" uid: facf56c6-f309-4e5d-8667-c2dd2f6fe3f1 kind: Event lastTimestamp: "2026-04-24T21:28:55Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1447" in 6.393s (6.393s including waiting). Image size: 1560926130 bytes.' metadata: creationTimestamp: "2026-04-24T21:28:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:55Z" name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml.18a9683c26154996 namespace: kserve-ci-e2e-test resourceVersion: "14611" uid: 5d072c19-e476-45dd-9771-7d314a393537 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml namespace: kserve-ci-e2e-test resourceVersion: "14312" uid: facf56c6-f309-4e5d-8667-c2dd2f6fe3f1 kind: Event lastTimestamp: "2026-04-24T21:28:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:28:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:55Z" name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml.18a9683c2ba64c88 namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: 9d7bcef8-ad89-4baa-84db-2b31a28d8cfe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml namespace: kserve-ci-e2e-test resourceVersion: "14312" uid: facf56c6-f309-4e5d-8667-c2dd2f6fe3f1 kind: Event lastTimestamp: "2026-04-24T21:28:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:28:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:55Z" name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml.18a9683c2c275ec3 namespace: kserve-ci-e2e-test resourceVersion: "14615" uid: 5ed5949d-c420-4f15-afc5-f975352c9e2a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml namespace: kserve-ci-e2e-test resourceVersion: "14312" uid: facf56c6-f309-4e5d-8667-c2dd2f6fe3f1 kind: Event lastTimestamp: "2026-04-24T21:28:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:28:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:55Z" name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml.18a9683c2c32a5b5 namespace: kserve-ci-e2e-test resourceVersion: "14616" uid: ece42687-fb00-4007-9672-b06b1ba9bfbf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml namespace: kserve-ci-e2e-test resourceVersion: "14312" uid: facf56c6-f309-4e5d-8667-c2dd2f6fe3f1 kind: Event lastTimestamp: "2026-04-24T21:28:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:28:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:55Z" name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml.18a9683c2f8e1ee7 namespace: kserve-ci-e2e-test resourceVersion: "14621" uid: 74802a95-68ba-4a2f-b57f-a78a3f2ee8f0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml namespace: kserve-ci-e2e-test resourceVersion: "14312" uid: facf56c6-f309-4e5d-8667-c2dd2f6fe3f1 kind: Event lastTimestamp: "2026-04-24T21:28:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:28:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:55Z" name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml.18a9683c301df408 namespace: kserve-ci-e2e-test resourceVersion: "14625" uid: f2dc2f33-d25f-4200-b86c-42ef12d9875f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:28:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml namespace: kserve-ci-e2e-test resourceVersion: "14312" uid: facf56c6-f309-4e5d-8667-c2dd2f6fe3f1 kind: Event lastTimestamp: "2026-04-24T21:30:42Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:28:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:42Z" name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml.18a9683c4bd37276 namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 714d49d0-775b-42a8-b1dd-9b6b700e9515 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml namespace: kserve-ci-e2e-test resourceVersion: "14312" uid: facf56c6-f309-4e5d-8667-c2dd2f6fe3f1 kind: Event lastTimestamp: "2026-04-24T21:30:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:30:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:38Z" name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml.18a968543c7a3d4b namespace: kserve-ci-e2e-test resourceVersion: "15815" uid: a82816b6-70dd-4c68-a1d3-d3154b1c8b85 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml namespace: kserve-ci-e2e-test resourceVersion: "14312" uid: facf56c6-f309-4e5d-8667-c2dd2f6fe3f1 kind: Event lastTimestamp: "2026-04-24T21:30:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:30:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:38Z" name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml.18a968543c7b58d3 namespace: kserve-ci-e2e-test resourceVersion: "15817" uid: c3958634-9a24-426b-9d76-9bb27193d3cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml namespace: kserve-ci-e2e-test resourceVersion: "14312" uid: facf56c6-f309-4e5d-8667-c2dd2f6fe3f1 kind: Event lastTimestamp: "2026-04-24T21:30:42Z" message: 'Readiness probe failed: Get "https://10.133.0.28:8643/healthz": dial tcp 10.133.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:42Z" name: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml.18a96854fa299a23 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: 1817cba1-4aff-4cbc-91a7-ab4373b4ff34 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-81.ec2.internal source: component: kubelet host: ip-10-0-132-81.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-87d6c5875 namespace: kserve-ci-e2e-test resourceVersion: "14296" uid: 14a1c8a3-7bfb-426f-b920-35aaa2141914 kind: Event lastTimestamp: "2026-04-24T21:28:29Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-87d6c5875-x4cml' metadata: creationTimestamp: "2026-04-24T21:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:28:29Z" name: isvc-sklearn-graph-2-predictor-87d6c5875.18a9683633deb4cb namespace: kserve-ci-e2e-test resourceVersion: "14311" uid: daffcede-f7cb-4053-b36b-847a42eba0dc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14294" uid: 4d725665-9b67-4624-93f5-fd24676ef409 kind: Event lastTimestamp: "2026-04-24T21:28:29Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-87d6c5875 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:28:29Z" name: isvc-sklearn-graph-2-predictor.18a968361d6909ae namespace: kserve-ci-e2e-test resourceVersion: "14297" uid: 2a466bb0-ef3c-4631-8255-81ca16c013e5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:28:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: 095f7c71-62b0-401f-8bd5-9dc11a805eca kind: Event lastTimestamp: "2026-04-24T21:29:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:29:29Z" name: isvc-sklearn-graph-2-predictor.18a968399be86a01 namespace: kserve-ci-e2e-test resourceVersion: "14918" uid: 915461ef-56b5-4a09-aa25-e54da0f0095a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:28:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: 095f7c71-62b0-401f-8bd5-9dc11a805eca kind: Event lastTimestamp: "2026-04-24T21:29:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:29:29Z" name: isvc-sklearn-graph-2-predictor.18a968399c4cd223 namespace: kserve-ci-e2e-test resourceVersion: "14920" uid: 54156a8b-e92f-4905-970c-344c265c81de reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:29:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14487" uid: 095f7c71-62b0-401f-8bd5-9dc11a805eca kind: Event lastTimestamp: "2026-04-24T21:30:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:29:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:29Z" name: isvc-sklearn-graph-2-predictor.18a9684797360895 namespace: kserve-ci-e2e-test resourceVersion: "15663" uid: 2f659c98-a7cf-4063-9150-421961ed6831 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:29:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14487" uid: 095f7c71-62b0-401f-8bd5-9dc11a805eca kind: Event lastTimestamp: "2026-04-24T21:30:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:29:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:29Z" name: isvc-sklearn-graph-2-predictor.18a9684797957a3f namespace: kserve-ci-e2e-test resourceVersion: "15664" uid: 0cb870bd-0be5-4540-b6c4-7f3cd5de8c40 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:28:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14266" uid: 645f368a-4815-43e5-9623-9ecd0488eb9b kind: Event lastTimestamp: "2026-04-24T21:30:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:30:02Z" name: isvc-sklearn-graph-2.18a968361d454b12 namespace: kserve-ci-e2e-test resourceVersion: "15253" uid: fc8b54ce-8eee-4cc4-b5e2-48e8827570a7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15248" uid: 645f368a-4815-43e5-9623-9ecd0488eb9b kind: Event lastTimestamp: "2026-04-24T21:30:02Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-04-24T21:30:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:30:02Z" name: isvc-sklearn-graph-2.18a9684baeea3f23 namespace: kserve-ci-e2e-test resourceVersion: "15249" uid: 1e2f59a3-1993-49c0-a714-ae6c257e70e9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:28:29.389272Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4 namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 45ff4ebd-30bb-4cf0-984b-e93e5a155663 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-669d8d6456-vw4f4 to ip-10-0-128-21.ec2.internal metadata: creationTimestamp: "2026-04-24T21:28:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:28:29Z" name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4.18a96836108894f8 namespace: kserve-ci-e2e-test resourceVersion: "14283" uid: 1c60ddd5-7c04-45ed-89a8-30bd83d63ddc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:29Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4 namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: 45ff4ebd-30bb-4cf0-984b-e93e5a155663 kind: Event lastTimestamp: "2026-04-24T21:28:29Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:28:29Z" name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4.18a968362da35fd6 namespace: kserve-ci-e2e-test resourceVersion: "14307" uid: 9c85a544-441d-474b-80b7-bd43ab330780 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4 namespace: kserve-ci-e2e-test resourceVersion: "14281" uid: 45ff4ebd-30bb-4cf0-984b-e93e5a155663 kind: Event lastTimestamp: "2026-04-24T21:28:29Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:85f85c16414082de64de782f602228652c10c5bace08b518e803a7e911c62e5e" metadata: creationTimestamp: "2026-04-24T21:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:29Z" name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4.18a96836302803da namespace: kserve-ci-e2e-test resourceVersion: "14309" uid: 080db0c7-fdc5-4eb0-8ed7-c47a86883a7a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4 namespace: kserve-ci-e2e-test resourceVersion: "14281" uid: 45ff4ebd-30bb-4cf0-984b-e93e5a155663 kind: Event lastTimestamp: "2026-04-24T21:28:42Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:85f85c16414082de64de782f602228652c10c5bace08b518e803a7e911c62e5e" in 12.108s (12.108s including waiting). Image size: 299849144 bytes.' metadata: creationTimestamp: "2026-04-24T21:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:42Z" name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4.18a9683901e834e3 namespace: kserve-ci-e2e-test resourceVersion: "14434" uid: 929fb865-7acb-4cb9-a003-f569e6bcd67c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4 namespace: kserve-ci-e2e-test resourceVersion: "14281" uid: 45ff4ebd-30bb-4cf0-984b-e93e5a155663 kind: Event lastTimestamp: "2026-04-24T21:28:42Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T21:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:42Z" name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4.18a9683904bbabaf namespace: kserve-ci-e2e-test resourceVersion: "14436" uid: 48ca41a4-94ee-4cb7-837a-fd239f783308 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4 namespace: kserve-ci-e2e-test resourceVersion: "14281" uid: 45ff4ebd-30bb-4cf0-984b-e93e5a155663 kind: Event lastTimestamp: "2026-04-24T21:28:42Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T21:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:42Z" name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4.18a96839052d76b2 namespace: kserve-ci-e2e-test resourceVersion: "14437" uid: 7bb853e1-c7e2-4eb7-9c74-8ac53c8ba3c0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4 namespace: kserve-ci-e2e-test resourceVersion: "14281" uid: 45ff4ebd-30bb-4cf0-984b-e93e5a155663 kind: Event lastTimestamp: "2026-04-24T21:28:46Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-24T21:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:46Z" name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4.18a9683a0c9d9d18 namespace: kserve-ci-e2e-test resourceVersion: "14523" uid: 1789c55f-b5cd-4ca2-a1fb-bcb3611ba1b2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4 namespace: kserve-ci-e2e-test resourceVersion: "14281" uid: 45ff4ebd-30bb-4cf0-984b-e93e5a155663 kind: Event lastTimestamp: "2026-04-24T21:29:05Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 19.329s (19.329s including waiting). Image size: 1306417402 bytes.' metadata: creationTimestamp: "2026-04-24T21:29:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:29:05Z" name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4.18a9683e8cb9ab20 namespace: kserve-ci-e2e-test resourceVersion: "14717" uid: 38eeb55c-a06f-455a-a0f5-eb1b32a0a885 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4 namespace: kserve-ci-e2e-test resourceVersion: "14281" uid: 45ff4ebd-30bb-4cf0-984b-e93e5a155663 kind: Event lastTimestamp: "2026-04-24T21:29:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:29:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:29:05Z" name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4.18a9683e8fd9eaed namespace: kserve-ci-e2e-test resourceVersion: "14718" uid: 8a938370-a934-498b-b69a-d8275dc0d493 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4 namespace: kserve-ci-e2e-test resourceVersion: "14281" uid: 45ff4ebd-30bb-4cf0-984b-e93e5a155663 kind: Event lastTimestamp: "2026-04-24T21:29:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:29:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:29:05Z" name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4.18a9683e905231c3 namespace: kserve-ci-e2e-test resourceVersion: "14719" uid: 28c6f810-1d83-41e4-b491-8ab0ba92d1e4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4 namespace: kserve-ci-e2e-test resourceVersion: "14281" uid: 45ff4ebd-30bb-4cf0-984b-e93e5a155663 kind: Event lastTimestamp: "2026-04-24T21:29:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:29:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:29:05Z" name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4.18a9683e90608d98 namespace: kserve-ci-e2e-test resourceVersion: "14720" uid: 3385f79e-0e9a-4bdc-a670-8179c085df69 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4 namespace: kserve-ci-e2e-test resourceVersion: "14281" uid: 45ff4ebd-30bb-4cf0-984b-e93e5a155663 kind: Event lastTimestamp: "2026-04-24T21:29:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:29:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:29:05Z" name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4.18a9683e93c3267e namespace: kserve-ci-e2e-test resourceVersion: "14721" uid: 5bd36032-3cbd-4771-b8ee-f0aeddc7da60 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4 namespace: kserve-ci-e2e-test resourceVersion: "14281" uid: 45ff4ebd-30bb-4cf0-984b-e93e5a155663 kind: Event lastTimestamp: "2026-04-24T21:29:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:29:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:29:05Z" name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4.18a9683e943287e9 namespace: kserve-ci-e2e-test resourceVersion: "14722" uid: e5db7c06-92a8-499f-8fa2-0612a95dbda1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T21:29:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4 namespace: kserve-ci-e2e-test resourceVersion: "14281" uid: 45ff4ebd-30bb-4cf0-984b-e93e5a155663 kind: Event lastTimestamp: "2026-04-24T21:30:42Z" message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:29:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:42Z" name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4.18a9683fe2b62c59 namespace: kserve-ci-e2e-test resourceVersion: "15942" uid: 71f95338-7291-41c6-97c5-df5a7130d25e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4 namespace: kserve-ci-e2e-test resourceVersion: "14281" uid: 45ff4ebd-30bb-4cf0-984b-e93e5a155663 kind: Event lastTimestamp: "2026-04-24T21:30:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:30:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:38Z" name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4.18a96854395d1555 namespace: kserve-ci-e2e-test resourceVersion: "15785" uid: 01334ce5-02c0-46e9-9159-6c2d92229bbc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4 namespace: kserve-ci-e2e-test resourceVersion: "14281" uid: 45ff4ebd-30bb-4cf0-984b-e93e5a155663 kind: Event lastTimestamp: "2026-04-24T21:30:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:30:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:38Z" name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4.18a96854395da31c namespace: kserve-ci-e2e-test resourceVersion: "15787" uid: 88b27441-5675-4c4d-a1f1-2f2f0e0e0d03 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4 namespace: kserve-ci-e2e-test resourceVersion: "14281" uid: 45ff4ebd-30bb-4cf0-984b-e93e5a155663 kind: Event lastTimestamp: "2026-04-24T21:30:41Z" message: 'Readiness probe failed: Get "https://10.132.0.28:8643/healthz": dial tcp 10.132.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:30:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:41Z" name: isvc-xgboost-graph-predictor-669d8d6456-vw4f4.18a96854d767334f namespace: kserve-ci-e2e-test resourceVersion: "15928" uid: 48d3091b-b97c-49ad-9b87-9e77f67dcc43 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-669d8d6456 namespace: kserve-ci-e2e-test resourceVersion: "14258" uid: d7d9a543-cf83-4473-8ed5-cacaa73b49eb kind: Event lastTimestamp: "2026-04-24T21:28:29Z" message: 'Created pod: isvc-xgboost-graph-predictor-669d8d6456-vw4f4' metadata: creationTimestamp: "2026-04-24T21:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:28:29Z" name: isvc-xgboost-graph-predictor-669d8d6456.18a968360ffe58eb namespace: kserve-ci-e2e-test resourceVersion: "14280" uid: a6e81df9-0cd2-4e07-a14e-eccec2fd78a4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14256" uid: a0e991a3-8f2d-42de-bb36-257a6b9c8ff9 kind: Event lastTimestamp: "2026-04-24T21:28:29Z" message: Scaled up replica set isvc-xgboost-graph-predictor-669d8d6456 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:28:29Z" name: isvc-xgboost-graph-predictor.18a96835ff22ab90 namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: bee14695-050f-4ebd-a52c-a5a0ae44f1f6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:28:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 4f56f80f-6e8a-4cbf-8461-19d3b41e5df6 kind: Event lastTimestamp: "2026-04-24T21:30:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:29Z" name: isvc-xgboost-graph-predictor.18a968397d8d55c6 namespace: kserve-ci-e2e-test resourceVersion: "15657" uid: 6cdb7bbc-03ef-46f2-9297-d03c43113824 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:28:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 4f56f80f-6e8a-4cbf-8461-19d3b41e5df6 kind: Event lastTimestamp: "2026-04-24T21:30:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:29Z" name: isvc-xgboost-graph-predictor.18a968397df325d7 namespace: kserve-ci-e2e-test resourceVersion: "15658" uid: 4ae3be52-d173-40ac-93f7-dbb41b94b6e7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: 4f56f80f-6e8a-4cbf-8461-19d3b41e5df6 kind: Event lastTimestamp: "2026-04-24T21:30:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:30:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:14Z" name: isvc-xgboost-graph-predictor.18a9684e7636f31d namespace: kserve-ci-e2e-test resourceVersion: "15503" uid: 78beb02f-27a7-4c63-98b0-90b5dfd03b10 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: 4f56f80f-6e8a-4cbf-8461-19d3b41e5df6 kind: Event lastTimestamp: "2026-04-24T21:30:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:30:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:14Z" name: isvc-xgboost-graph-predictor.18a9684e76a0188f namespace: kserve-ci-e2e-test resourceVersion: "15505" uid: 03385bfe-d85b-433d-96ec-3469120aac00 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T21:28:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14236" uid: cdbde08c-f272-4ae2-a179-bb9b883cc9b3 kind: Event lastTimestamp: "2026-04-24T21:30:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:30:12Z" name: isvc-xgboost-graph.18a96835fefcfd18 namespace: kserve-ci-e2e-test resourceVersion: "15486" uid: 4f981e17-7bcb-49df-bf01-d52b7fa56b68 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15484" uid: cdbde08c-f272-4ae2-a179-bb9b883cc9b3 kind: Event lastTimestamp: "2026-04-24T21:30:12Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-04-24T21:30:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:30:12Z" name: isvc-xgboost-graph.18a9684e1c684d87 namespace: kserve-ci-e2e-test resourceVersion: "15485" uid: aa421e4a-0316-4a96-9c1a-ad0677dc10c5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:30:28.764792Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-bfd66c7dd-dqj6x namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: 9ddb90b6-48de-4a26-87be-bafd5c0a59aa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-bfd66c7dd-dqj6x to ip-10-0-128-21.ec2.internal metadata: creationTimestamp: "2026-04-24T21:30:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:30:28Z" name: model-chainer-bfd66c7dd-dqj6x.18a96851dbde742d namespace: kserve-ci-e2e-test resourceVersion: "15637" uid: c0ea9140-e269-4a80-8631-46d251f68c07 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:28Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-bfd66c7dd-dqj6x namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 9ddb90b6-48de-4a26-87be-bafd5c0a59aa kind: Event lastTimestamp: "2026-04-24T21:30:28Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:30:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:28Z" name: model-chainer-bfd66c7dd-dqj6x.18a96851e6168135 namespace: kserve-ci-e2e-test resourceVersion: "15649" uid: 7cd85318-7035-49d8-bdc0-0207c68e4200 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:29Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-bfd66c7dd-dqj6x namespace: kserve-ci-e2e-test resourceVersion: "15641" uid: 9ddb90b6-48de-4a26-87be-bafd5c0a59aa kind: Event lastTimestamp: "2026-04-24T21:30:29Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:30:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:30:29Z" name: model-chainer-bfd66c7dd-dqj6x.18a9685218c4f4f3 namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 308ee5ea-d329-4973-9427-3177303b222b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-bfd66c7dd-dqj6x namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 9ddb90b6-48de-4a26-87be-bafd5c0a59aa kind: Event lastTimestamp: "2026-04-24T21:30:29Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f2021b9f79dbd151723ae4c386dd28bedd0fff23f6daa0825bdc58598fc71887" already present on machine metadata: creationTimestamp: "2026-04-24T21:30:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:29Z" name: model-chainer-bfd66c7dd-dqj6x.18a9685219f6baa9 namespace: kserve-ci-e2e-test resourceVersion: "15667" uid: b581bf7c-79df-4cfb-851a-9dc164ed9e83 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-bfd66c7dd-dqj6x namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 9ddb90b6-48de-4a26-87be-bafd5c0a59aa kind: Event lastTimestamp: "2026-04-24T21:30:29Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-04-24T21:30:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:29Z" name: model-chainer-bfd66c7dd-dqj6x.18a968521d695c06 namespace: kserve-ci-e2e-test resourceVersion: "15668" uid: 4918348a-1785-44de-b205-7f6abc356404 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-bfd66c7dd-dqj6x namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 9ddb90b6-48de-4a26-87be-bafd5c0a59aa kind: Event lastTimestamp: "2026-04-24T21:30:29Z" message: Started container model-chainer metadata: creationTimestamp: "2026-04-24T21:30:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:29Z" name: model-chainer-bfd66c7dd-dqj6x.18a968521ddfcc0e namespace: kserve-ci-e2e-test resourceVersion: "15669" uid: f0fd277d-a7d9-4115-8b55-47d4a1ff3978 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-bfd66c7dd-dqj6x namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 9ddb90b6-48de-4a26-87be-bafd5c0a59aa kind: Event lastTimestamp: "2026-04-24T21:30:38Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-04-24T21:30:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:38Z" name: model-chainer-bfd66c7dd-dqj6x.18a9685434e6df08 namespace: kserve-ci-e2e-test resourceVersion: "15764" uid: bd0ae9e9-6b04-495a-8fbb-46feebfeec91 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:30:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-bfd66c7dd-dqj6x namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 9ddb90b6-48de-4a26-87be-bafd5c0a59aa kind: Event lastTimestamp: "2026-04-24T21:31:06Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:30:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:31:06Z" name: model-chainer-bfd66c7dd-dqj6x.18a96854e76ce0c6 namespace: kserve-ci-e2e-test resourceVersion: "16198" uid: 8da891e2-5fd7-4607-98bf-afe32c977e09 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-bfd66c7dd namespace: kserve-ci-e2e-test resourceVersion: "15629" uid: 6c232a47-2d51-4828-bfc3-a450f4fcdf50 kind: Event lastTimestamp: "2026-04-24T21:30:28Z" message: 'Created pod: model-chainer-bfd66c7dd-dqj6x' metadata: creationTimestamp: "2026-04-24T21:30:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:28Z" name: model-chainer-bfd66c7dd.18a96851db4dc177 namespace: kserve-ci-e2e-test resourceVersion: "15633" uid: 951cae09-e638-477e-b09f-d161684b424e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15627" uid: 0c530cc3-96fc-42c4-aa7c-91f2dc5adcf4 kind: Event lastTimestamp: "2026-04-24T21:30:28Z" message: Scaled up replica set model-chainer-bfd66c7dd from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:30:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:28Z" name: model-chainer.18a96851da9137e0 namespace: kserve-ci-e2e-test resourceVersion: "15630" uid: 52fc79c5-7a4d-4eeb-bd38-b8ae2b5648a3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:28Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15644" uid: ec4daea7-6615-49ea-8cb1-08dd0ae56aef kind: Event lastTimestamp: "2026-04-24T21:30:28Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-04-24T21:30:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:30:28Z" name: model-chainer.18a96851de3bd3c5 namespace: kserve-ci-e2e-test resourceVersion: "15646" uid: fc278bf3-c11e-42eb-bc5d-b6598e71dfab reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:31:39.014188Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-3a238-f58988f4b-8hmrl namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: e8bfc6b6-00e2-4dd7-9df2-44f98dbb24d2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-3a238-f58988f4b-8hmrl to ip-10-0-128-21.ec2.internal metadata: creationTimestamp: "2026-04-24T21:31:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:31:39Z" name: sequence-graph-3a238-f58988f4b-8hmrl.18a96862370f2732 namespace: kserve-ci-e2e-test resourceVersion: "16492" uid: bcbba8e9-1467-4f52-bd62-35f63c981bc7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:31:39Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-3a238-f58988f4b-8hmrl namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: e8bfc6b6-00e2-4dd7-9df2-44f98dbb24d2 kind: Event lastTimestamp: "2026-04-24T21:31:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-3a238-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:31:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:31:39Z" name: sequence-graph-3a238-f58988f4b-8hmrl.18a9686242ff3d75 namespace: kserve-ci-e2e-test resourceVersion: "16506" uid: 64caf6d5-3ca7-4723-b366-ee54c193dcd3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:31:40Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-3a238-f58988f4b-8hmrl namespace: kserve-ci-e2e-test resourceVersion: "16498" uid: e8bfc6b6-00e2-4dd7-9df2-44f98dbb24d2 kind: Event lastTimestamp: "2026-04-24T21:31:40Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:31:40Z" name: sequence-graph-3a238-f58988f4b-8hmrl.18a9686273f91267 namespace: kserve-ci-e2e-test resourceVersion: "16516" uid: 74e9a2a2-6e16-462a-b980-addb80872141 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:31:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3a238} kind: Pod name: sequence-graph-3a238-f58988f4b-8hmrl namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: e8bfc6b6-00e2-4dd7-9df2-44f98dbb24d2 kind: Event lastTimestamp: "2026-04-24T21:31:40Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f2021b9f79dbd151723ae4c386dd28bedd0fff23f6daa0825bdc58598fc71887" already present on machine metadata: creationTimestamp: "2026-04-24T21:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:31:40Z" name: sequence-graph-3a238-f58988f4b-8hmrl.18a96862754ca996 namespace: kserve-ci-e2e-test resourceVersion: "16519" uid: 3f8412ac-1e51-42d8-a272-327a586d9414 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:31:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3a238} kind: Pod name: sequence-graph-3a238-f58988f4b-8hmrl namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: e8bfc6b6-00e2-4dd7-9df2-44f98dbb24d2 kind: Event lastTimestamp: "2026-04-24T21:31:40Z" message: 'Created container: sequence-graph-3a238' metadata: creationTimestamp: "2026-04-24T21:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:31:40Z" name: sequence-graph-3a238-f58988f4b-8hmrl.18a96862788b2938 namespace: kserve-ci-e2e-test resourceVersion: "16521" uid: 230f3efb-812e-4d17-b923-147400a60732 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:31:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3a238} kind: Pod name: sequence-graph-3a238-f58988f4b-8hmrl namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: e8bfc6b6-00e2-4dd7-9df2-44f98dbb24d2 kind: Event lastTimestamp: "2026-04-24T21:31:40Z" message: Started container sequence-graph-3a238 metadata: creationTimestamp: "2026-04-24T21:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:31:40Z" name: sequence-graph-3a238-f58988f4b-8hmrl.18a96862790141ad namespace: kserve-ci-e2e-test resourceVersion: "16522" uid: 46802448-0255-4160-95fb-719f3084b1a7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3a238} kind: Pod name: sequence-graph-3a238-f58988f4b-8hmrl namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: e8bfc6b6-00e2-4dd7-9df2-44f98dbb24d2 kind: Event lastTimestamp: "2026-04-24T21:39:53Z" message: Stopping container sequence-graph-3a238 metadata: creationTimestamp: "2026-04-24T21:39:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:53Z" name: sequence-graph-3a238-f58988f4b-8hmrl.18a968d56865f4b6 namespace: kserve-ci-e2e-test resourceVersion: "20095" uid: fc84da02-8458-439a-a609-9fa444bc2d90 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:39:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3a238} kind: Pod name: sequence-graph-3a238-f58988f4b-8hmrl namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: e8bfc6b6-00e2-4dd7-9df2-44f98dbb24d2 kind: Event lastTimestamp: "2026-04-24T21:40:22Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:39:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:22Z" name: sequence-graph-3a238-f58988f4b-8hmrl.18a968d62e45f1ec namespace: kserve-ci-e2e-test resourceVersion: "20502" uid: 5f20283f-5ff6-4885-a376-c46320ff5324 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:31:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-3a238-f58988f4b namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: 2ae151c0-3339-4750-83c7-482cb7da0aab kind: Event lastTimestamp: "2026-04-24T21:31:39Z" message: 'Created pod: sequence-graph-3a238-f58988f4b-8hmrl' metadata: creationTimestamp: "2026-04-24T21:31:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:31:39Z" name: sequence-graph-3a238-f58988f4b.18a9686236a25ea0 namespace: kserve-ci-e2e-test resourceVersion: "16488" uid: 102a2fbc-c64e-4d06-8cd4-ecc9fb702935 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:31:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-3a238 namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: 56227fee-7a0e-4e68-9fd2-d3a1b5c4b1b0 kind: Event lastTimestamp: "2026-04-24T21:31:38Z" message: Scaled up replica set sequence-graph-3a238-f58988f4b from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:31:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:31:38Z" name: sequence-graph-3a238.18a9686235eed599 namespace: kserve-ci-e2e-test resourceVersion: "16484" uid: 86389779-b96c-4b6d-82b0-7ef03958dd8b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:31:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-3a238 namespace: kserve-ci-e2e-test resourceVersion: "16500" uid: 38fd1ead-3b88-4e13-a9a7-c25451cac60b kind: Event lastTimestamp: "2026-04-24T21:31:39Z" message: InferenceGraph [sequence-graph-3a238] is Ready metadata: creationTimestamp: "2026-04-24T21:31:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:31:39Z" name: sequence-graph-3a238.18a968623a68ea82 namespace: kserve-ci-e2e-test resourceVersion: "16502" uid: efdb6c54-e507-4743-a817-97746f3be192 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:31:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-3a238 namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: 1eb944b1-f4ee-488a-8b1c-39ee012c99ab kind: Event lastTimestamp: "2026-04-24T21:32:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:31:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:32:09Z" name: sequence-graph-3a238.18a96865b4726d90 namespace: kserve-ci-e2e-test resourceVersion: "16739" uid: 575c9b9d-8ce4-4198-9fa9-197151940bf4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:31:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-3a238 namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: 1eb944b1-f4ee-488a-8b1c-39ee012c99ab kind: Event lastTimestamp: "2026-04-24T21:32:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:31:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:32:09Z" name: sequence-graph-3a238.18a96865b4d264bf namespace: kserve-ci-e2e-test resourceVersion: "16740" uid: ed784293-059a-440d-a295-8cfe8cc2f417 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:42:04.175206Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-64806-67786fdb9f-r75d4 namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: 2d311244-c2f3-4350-a0d4-51588e56c67c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-64806-67786fdb9f-r75d4 to ip-10-0-128-21.ec2.internal metadata: creationTimestamp: "2026-04-24T21:42:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:42:04Z" name: sequence-graph-64806-67786fdb9f-r75d4.18a968f3c58f47c5 namespace: kserve-ci-e2e-test resourceVersion: "21753" uid: c10610bb-c065-418e-bbf5-4439e13222c5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:42:04Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-64806-67786fdb9f-r75d4 namespace: kserve-ci-e2e-test resourceVersion: "21749" uid: 2d311244-c2f3-4350-a0d4-51588e56c67c kind: Event lastTimestamp: "2026-04-24T21:42:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-64806-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:42:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:42:04Z" name: sequence-graph-64806-67786fdb9f-r75d4.18a968f3cf8672ca namespace: kserve-ci-e2e-test resourceVersion: "21766" uid: 3e162e81-7a40-41de-a5ee-a64488f5e543 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:42:05Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-64806-67786fdb9f-r75d4 namespace: kserve-ci-e2e-test resourceVersion: "21757" uid: 2d311244-c2f3-4350-a0d4-51588e56c67c kind: Event lastTimestamp: "2026-04-24T21:42:05Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:42:05Z" name: sequence-graph-64806-67786fdb9f-r75d4.18a968f4028a10f3 namespace: kserve-ci-e2e-test resourceVersion: "21774" uid: d530067b-a98a-4b83-8144-affb28c84532 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:42:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-64806} kind: Pod name: sequence-graph-64806-67786fdb9f-r75d4 namespace: kserve-ci-e2e-test resourceVersion: "21749" uid: 2d311244-c2f3-4350-a0d4-51588e56c67c kind: Event lastTimestamp: "2026-04-24T21:42:05Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f2021b9f79dbd151723ae4c386dd28bedd0fff23f6daa0825bdc58598fc71887" already present on machine metadata: creationTimestamp: "2026-04-24T21:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:42:05Z" name: sequence-graph-64806-67786fdb9f-r75d4.18a968f403e3c139 namespace: kserve-ci-e2e-test resourceVersion: "21776" uid: a2553158-74ba-4ffe-8cec-36bb85d792fe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:42:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-64806} kind: Pod name: sequence-graph-64806-67786fdb9f-r75d4 namespace: kserve-ci-e2e-test resourceVersion: "21749" uid: 2d311244-c2f3-4350-a0d4-51588e56c67c kind: Event lastTimestamp: "2026-04-24T21:42:05Z" message: 'Created container: sequence-graph-64806' metadata: creationTimestamp: "2026-04-24T21:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:42:05Z" name: sequence-graph-64806-67786fdb9f-r75d4.18a968f40753d38b namespace: kserve-ci-e2e-test resourceVersion: "21779" uid: 442d6473-5c06-41cd-aa3b-1aa620d04eaf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:42:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-64806} kind: Pod name: sequence-graph-64806-67786fdb9f-r75d4 namespace: kserve-ci-e2e-test resourceVersion: "21749" uid: 2d311244-c2f3-4350-a0d4-51588e56c67c kind: Event lastTimestamp: "2026-04-24T21:42:05Z" message: Started container sequence-graph-64806 metadata: creationTimestamp: "2026-04-24T21:42:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:42:05Z" name: sequence-graph-64806-67786fdb9f-r75d4.18a968f407c43b3c namespace: kserve-ci-e2e-test resourceVersion: "21780" uid: 4d6bad7a-713c-40d7-88b2-66295d1fe9f4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-64806} kind: Pod name: sequence-graph-64806-67786fdb9f-r75d4 namespace: kserve-ci-e2e-test resourceVersion: "21749" uid: 2d311244-c2f3-4350-a0d4-51588e56c67c kind: Event lastTimestamp: "2026-04-24T21:50:18Z" message: Stopping container sequence-graph-64806 metadata: creationTimestamp: "2026-04-24T21:50:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:18Z" name: sequence-graph-64806-67786fdb9f-r75d4.18a96966f3781564 namespace: kserve-ci-e2e-test resourceVersion: "25619" uid: 00cae0f7-9e36-44f0-a33e-997e733b4160 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:50:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-64806} kind: Pod name: sequence-graph-64806-67786fdb9f-r75d4 namespace: kserve-ci-e2e-test resourceVersion: "21749" uid: 2d311244-c2f3-4350-a0d4-51588e56c67c kind: Event lastTimestamp: "2026-04-24T21:50:47Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:50:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:47Z" name: sequence-graph-64806-67786fdb9f-r75d4.18a96967b7c8de36 namespace: kserve-ci-e2e-test resourceVersion: "26026" uid: cc88fe2d-6529-4103-ba63-52be672e1ff3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:42:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-64806-67786fdb9f namespace: kserve-ci-e2e-test resourceVersion: "21743" uid: d219b7a5-9507-46ab-8c70-9df192380d49 kind: Event lastTimestamp: "2026-04-24T21:42:04Z" message: 'Created pod: sequence-graph-64806-67786fdb9f-r75d4' metadata: creationTimestamp: "2026-04-24T21:42:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:42:04Z" name: sequence-graph-64806-67786fdb9f.18a968f3c526ddc8 namespace: kserve-ci-e2e-test resourceVersion: "21748" uid: 7d35efb4-8ab1-4fc7-a8b5-9f3647104677 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:42:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-64806 namespace: kserve-ci-e2e-test resourceVersion: "21741" uid: 6632e9d9-0ff8-4a69-bd26-4268b8b0369b kind: Event lastTimestamp: "2026-04-24T21:42:04Z" message: Scaled up replica set sequence-graph-64806-67786fdb9f from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:42:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:42:04Z" name: sequence-graph-64806.18a968f3c479d728 namespace: kserve-ci-e2e-test resourceVersion: "21744" uid: 16d4dc53-ee46-4f7d-920b-2facfd4b503c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:42:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-64806 namespace: kserve-ci-e2e-test resourceVersion: "21760" uid: d0938434-36ed-45d4-8b91-1944b6d70538 kind: Event lastTimestamp: "2026-04-24T21:42:04Z" message: InferenceGraph [sequence-graph-64806] is Ready metadata: creationTimestamp: "2026-04-24T21:42:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:42:04Z" name: sequence-graph-64806.18a968f3c9308227 namespace: kserve-ci-e2e-test resourceVersion: "21762" uid: f3c866d1-79c3-4e7c-bdf7-46739e1879f8 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:42:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-64806 namespace: kserve-ci-e2e-test resourceVersion: "21742" uid: 578d3c05-9c94-4f24-8457-73fe9fa5e2bf kind: Event lastTimestamp: "2026-04-24T21:42:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:42:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:42:34Z" name: sequence-graph-64806.18a968f742f3185e namespace: kserve-ci-e2e-test resourceVersion: "21994" uid: a2a8b53c-a6be-4111-a484-4f8e3ba599d9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:42:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-64806 namespace: kserve-ci-e2e-test resourceVersion: "21742" uid: 578d3c05-9c94-4f24-8457-73fe9fa5e2bf kind: Event lastTimestamp: "2026-04-24T21:42:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:42:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:42:34Z" name: sequence-graph-64806.18a968f7433a0c29 namespace: kserve-ci-e2e-test resourceVersion: "21995" uid: ce54e4e8-4c1b-43db-9648-3e2050adaba1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:40:53.930347Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c1a35-5b75598d54-4l255 namespace: kserve-ci-e2e-test resourceVersion: "20932" uid: daa90fc4-db12-4619-8ad2-1823d103548e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-c1a35-5b75598d54-4l255 to ip-10-0-128-21.ec2.internal metadata: creationTimestamp: "2026-04-24T21:40:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:40:53Z" name: sequence-graph-c1a35-5b75598d54-4l255.18a968e36aa3da79 namespace: kserve-ci-e2e-test resourceVersion: "20937" uid: bb0e5f09-e156-4065-991c-64ebfa1a4e8e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:54Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c1a35-5b75598d54-4l255 namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: daa90fc4-db12-4619-8ad2-1823d103548e kind: Event lastTimestamp: "2026-04-24T21:40:54Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:40:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:40:54Z" name: sequence-graph-c1a35-5b75598d54-4l255.18a968e383fc3508 namespace: kserve-ci-e2e-test resourceVersion: "20958" uid: 923c7b21-0a0b-4c0e-a3c3-a567e6ec3939 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c1a35} kind: Pod name: sequence-graph-c1a35-5b75598d54-4l255 namespace: kserve-ci-e2e-test resourceVersion: "20934" uid: daa90fc4-db12-4619-8ad2-1823d103548e kind: Event lastTimestamp: "2026-04-24T21:40:54Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f2021b9f79dbd151723ae4c386dd28bedd0fff23f6daa0825bdc58598fc71887" already present on machine metadata: creationTimestamp: "2026-04-24T21:40:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:54Z" name: sequence-graph-c1a35-5b75598d54-4l255.18a968e385583680 namespace: kserve-ci-e2e-test resourceVersion: "20960" uid: 09a19ba4-b714-4ccf-bed1-930fca34043f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c1a35} kind: Pod name: sequence-graph-c1a35-5b75598d54-4l255 namespace: kserve-ci-e2e-test resourceVersion: "20934" uid: daa90fc4-db12-4619-8ad2-1823d103548e kind: Event lastTimestamp: "2026-04-24T21:40:54Z" message: 'Created container: sequence-graph-c1a35' metadata: creationTimestamp: "2026-04-24T21:40:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:54Z" name: sequence-graph-c1a35-5b75598d54-4l255.18a968e38889802c namespace: kserve-ci-e2e-test resourceVersion: "20961" uid: bf77eaef-c909-4395-9114-65666b872891 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c1a35} kind: Pod name: sequence-graph-c1a35-5b75598d54-4l255 namespace: kserve-ci-e2e-test resourceVersion: "20934" uid: daa90fc4-db12-4619-8ad2-1823d103548e kind: Event lastTimestamp: "2026-04-24T21:40:54Z" message: Started container sequence-graph-c1a35 metadata: creationTimestamp: "2026-04-24T21:40:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:54Z" name: sequence-graph-c1a35-5b75598d54-4l255.18a968e388fb1638 namespace: kserve-ci-e2e-test resourceVersion: "20962" uid: ccb2a698-16a9-44d2-b4e6-4fa256365f9f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c1a35} kind: Pod name: sequence-graph-c1a35-5b75598d54-4l255 namespace: kserve-ci-e2e-test resourceVersion: "20934" uid: daa90fc4-db12-4619-8ad2-1823d103548e kind: Event lastTimestamp: "2026-04-24T21:41:03Z" message: Stopping container sequence-graph-c1a35 metadata: creationTimestamp: "2026-04-24T21:41:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:03Z" name: sequence-graph-c1a35-5b75598d54-4l255.18a968e5c1f03766 namespace: kserve-ci-e2e-test resourceVersion: "21061" uid: 0cdbfa39-a65a-4ce5-a3b0-a3cffab85b8f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:41:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c1a35} kind: Pod name: sequence-graph-c1a35-5b75598d54-4l255 namespace: kserve-ci-e2e-test resourceVersion: "20934" uid: daa90fc4-db12-4619-8ad2-1823d103548e kind: Event lastTimestamp: "2026-04-24T21:41:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:41:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:30Z" name: sequence-graph-c1a35-5b75598d54-4l255.18a968e634b84a6d namespace: kserve-ci-e2e-test resourceVersion: "21455" uid: d8d79476-5b84-4025-81a0-42ab76063d71 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-c1a35-5b75598d54 namespace: kserve-ci-e2e-test resourceVersion: "20929" uid: e1cbe218-f7ab-453d-af78-13df381e2181 kind: Event lastTimestamp: "2026-04-24T21:40:53Z" message: 'Created pod: sequence-graph-c1a35-5b75598d54-4l255' metadata: creationTimestamp: "2026-04-24T21:40:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:53Z" name: sequence-graph-c1a35-5b75598d54.18a968e36a38f9ef namespace: kserve-ci-e2e-test resourceVersion: "20933" uid: 228dfa9f-7a5c-4a78-97f6-bdb00bdfffe1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-c1a35 namespace: kserve-ci-e2e-test resourceVersion: "20927" uid: cb66d677-c9cf-470f-b140-2b1205d02094 kind: Event lastTimestamp: "2026-04-24T21:40:53Z" message: Scaled up replica set sequence-graph-c1a35-5b75598d54 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:40:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:53Z" name: sequence-graph-c1a35.18a968e369922dff namespace: kserve-ci-e2e-test resourceVersion: "20930" uid: 39fed58a-3ac7-42e6-830e-837f9e282f12 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:53Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-c1a35 namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: b5bcb1fc-b7f0-4c87-82ef-e9e011cb90c3 kind: Event lastTimestamp: "2026-04-24T21:40:53Z" message: InferenceGraph [sequence-graph-c1a35] is Ready metadata: creationTimestamp: "2026-04-24T21:40:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:40:53Z" name: sequence-graph-c1a35.18a968e36d64030b namespace: kserve-ci-e2e-test resourceVersion: "20947" uid: efec714e-6ada-4e9d-8f65-f4e5be802ea4 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:51:53.414087Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-12404-554567d986-qpd8n namespace: kserve-ci-e2e-test resourceVersion: "26787" uid: 58718984-9436-4079-8b1d-dfe5d1aab879 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-12404-554567d986-qpd8n to ip-10-0-128-21.ec2.internal metadata: creationTimestamp: "2026-04-24T21:51:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:51:53Z" name: splitter-graph-12404-554567d986-qpd8n.18a9697cf6ef12fa namespace: kserve-ci-e2e-test resourceVersion: "26792" uid: b15c9128-392b-46b7-a6cb-e155861fb898 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:51:53Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-12404-554567d986-qpd8n namespace: kserve-ci-e2e-test resourceVersion: "26795" uid: 58718984-9436-4079-8b1d-dfe5d1aab879 kind: Event lastTimestamp: "2026-04-24T21:51:53Z" message: Add eth0 [10.132.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:51:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:51:53Z" name: splitter-graph-12404-554567d986-qpd8n.18a9697d1052a8c8 namespace: kserve-ci-e2e-test resourceVersion: "26809" uid: d1c5fe32-e55e-44ed-b18e-9772c3eb9f7a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:51:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-12404} kind: Pod name: splitter-graph-12404-554567d986-qpd8n namespace: kserve-ci-e2e-test resourceVersion: "26789" uid: 58718984-9436-4079-8b1d-dfe5d1aab879 kind: Event lastTimestamp: "2026-04-24T21:51:53Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f2021b9f79dbd151723ae4c386dd28bedd0fff23f6daa0825bdc58598fc71887" already present on machine metadata: creationTimestamp: "2026-04-24T21:51:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:51:53Z" name: splitter-graph-12404-554567d986-qpd8n.18a9697d11785766 namespace: kserve-ci-e2e-test resourceVersion: "26811" uid: 885b011f-5846-4c4d-816d-363c3d81928d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:51:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-12404} kind: Pod name: splitter-graph-12404-554567d986-qpd8n namespace: kserve-ci-e2e-test resourceVersion: "26789" uid: 58718984-9436-4079-8b1d-dfe5d1aab879 kind: Event lastTimestamp: "2026-04-24T21:51:53Z" message: 'Created container: splitter-graph-12404' metadata: creationTimestamp: "2026-04-24T21:51:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:51:53Z" name: splitter-graph-12404-554567d986-qpd8n.18a9697d14a56162 namespace: kserve-ci-e2e-test resourceVersion: "26812" uid: e7973417-2b6a-4174-a581-735eec8af769 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:51:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-12404} kind: Pod name: splitter-graph-12404-554567d986-qpd8n namespace: kserve-ci-e2e-test resourceVersion: "26789" uid: 58718984-9436-4079-8b1d-dfe5d1aab879 kind: Event lastTimestamp: "2026-04-24T21:51:53Z" message: Started container splitter-graph-12404 metadata: creationTimestamp: "2026-04-24T21:51:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:51:53Z" name: splitter-graph-12404-554567d986-qpd8n.18a9697d151849aa namespace: kserve-ci-e2e-test resourceVersion: "26813" uid: bc539ce9-c617-49f9-b1fb-a88b5f8d32cc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:00:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-12404} kind: Pod name: splitter-graph-12404-554567d986-qpd8n namespace: kserve-ci-e2e-test resourceVersion: "26789" uid: 58718984-9436-4079-8b1d-dfe5d1aab879 kind: Event lastTimestamp: "2026-04-24T22:00:08Z" message: Stopping container splitter-graph-12404 metadata: creationTimestamp: "2026-04-24T22:00:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:00:08Z" name: splitter-graph-12404-554567d986-qpd8n.18a969f0235bcecb namespace: kserve-ci-e2e-test resourceVersion: "30230" uid: dd67b160-72bd-4d8a-861e-3c972ba578ba reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T22:00:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-12404} kind: Pod name: splitter-graph-12404-554567d986-qpd8n namespace: kserve-ci-e2e-test resourceVersion: "26789" uid: 58718984-9436-4079-8b1d-dfe5d1aab879 kind: Event lastTimestamp: "2026-04-24T22:00:35Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T22:00:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:00:35Z" name: splitter-graph-12404-554567d986-qpd8n.18a969f09baac5c6 namespace: kserve-ci-e2e-test resourceVersion: "30466" uid: f4f0578b-432c-4ee2-a20e-a8cb5ac6040f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:51:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-12404-554567d986 namespace: kserve-ci-e2e-test resourceVersion: "26784" uid: c0c63201-657b-4c20-8fba-4272bfb5cd2a kind: Event lastTimestamp: "2026-04-24T21:51:53Z" message: 'Created pod: splitter-graph-12404-554567d986-qpd8n' metadata: creationTimestamp: "2026-04-24T21:51:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:51:53Z" name: splitter-graph-12404-554567d986.18a9697cf6875e56 namespace: kserve-ci-e2e-test resourceVersion: "26788" uid: 2864aa8b-dc04-40f0-a23a-15252569a996 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:51:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-12404 namespace: kserve-ci-e2e-test resourceVersion: "26782" uid: 4cf46e5a-59f0-4538-87f3-ad049cb8a2c7 kind: Event lastTimestamp: "2026-04-24T21:51:53Z" message: Scaled up replica set splitter-graph-12404-554567d986 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:51:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:51:53Z" name: splitter-graph-12404.18a9697cf5afae02 namespace: kserve-ci-e2e-test resourceVersion: "26785" uid: bab341f5-0e6e-4858-aee7-9de0796aa79a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:51:53Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-12404 namespace: kserve-ci-e2e-test resourceVersion: "26798" uid: 9767325f-8817-4554-9788-5f5eb5f8965f kind: Event lastTimestamp: "2026-04-24T21:51:53Z" message: InferenceGraph [splitter-graph-12404] is Ready metadata: creationTimestamp: "2026-04-24T21:51:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:51:53Z" name: splitter-graph-12404.18a9697cf9849348 namespace: kserve-ci-e2e-test resourceVersion: "26800" uid: 4220782f-c821-48c9-b119-3b257aec46e3 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:52:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-12404 namespace: kserve-ci-e2e-test resourceVersion: "26783" uid: 2cc049c4-bc10-4bf6-9fe4-e2416bcb95fe kind: Event lastTimestamp: "2026-04-24T21:52:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:52:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:52:23Z" name: splitter-graph-12404.18a969807434bfa9 namespace: kserve-ci-e2e-test resourceVersion: "27033" uid: 01e12e23-e5a7-4c5a-9fbf-8ff6b7ea4836 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:52:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-12404 namespace: kserve-ci-e2e-test resourceVersion: "26783" uid: 2cc049c4-bc10-4bf6-9fe4-e2416bcb95fe kind: Event lastTimestamp: "2026-04-24T21:52:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:52:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:52:23Z" name: splitter-graph-12404.18a969807498d35d namespace: kserve-ci-e2e-test resourceVersion: "27034" uid: e9175bb3-c416-471c-927d-c8227990e431 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:50:43.129992Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-4efea-5bb974bbff-2dr89 namespace: kserve-ci-e2e-test resourceVersion: "25971" uid: 7b374bd5-2f71-4887-b183-48690a08cf17 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-4efea-5bb974bbff-2dr89 to ip-10-0-128-21.ec2.internal metadata: creationTimestamp: "2026-04-24T21:50:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:50:43Z" name: splitter-graph-4efea-5bb974bbff-2dr89.18a9696c99ace0c5 namespace: kserve-ci-e2e-test resourceVersion: "25976" uid: 27042620-41b5-45f7-845b-79cc50ed5d62 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:43Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-4efea-5bb974bbff-2dr89 namespace: kserve-ci-e2e-test resourceVersion: "25980" uid: 7b374bd5-2f71-4887-b183-48690a08cf17 kind: Event lastTimestamp: "2026-04-24T21:50:43Z" message: Add eth0 [10.132.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:50:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:50:43Z" name: splitter-graph-4efea-5bb974bbff-2dr89.18a9696cb3674416 namespace: kserve-ci-e2e-test resourceVersion: "25993" uid: 4ad26aea-46ab-493c-ad3c-fbbf177c6935 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4efea} kind: Pod name: splitter-graph-4efea-5bb974bbff-2dr89 namespace: kserve-ci-e2e-test resourceVersion: "25973" uid: 7b374bd5-2f71-4887-b183-48690a08cf17 kind: Event lastTimestamp: "2026-04-24T21:50:43Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f2021b9f79dbd151723ae4c386dd28bedd0fff23f6daa0825bdc58598fc71887" already present on machine metadata: creationTimestamp: "2026-04-24T21:50:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:43Z" name: splitter-graph-4efea-5bb974bbff-2dr89.18a9696cb49817f1 namespace: kserve-ci-e2e-test resourceVersion: "25995" uid: 3bd3a8da-2f08-4191-96f4-a10a2ea0583b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4efea} kind: Pod name: splitter-graph-4efea-5bb974bbff-2dr89 namespace: kserve-ci-e2e-test resourceVersion: "25973" uid: 7b374bd5-2f71-4887-b183-48690a08cf17 kind: Event lastTimestamp: "2026-04-24T21:50:43Z" message: 'Created container: splitter-graph-4efea' metadata: creationTimestamp: "2026-04-24T21:50:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:43Z" name: splitter-graph-4efea-5bb974bbff-2dr89.18a9696cb826557e namespace: kserve-ci-e2e-test resourceVersion: "25996" uid: fa4497fb-add7-4e7f-8cf9-aa81d0f8b1e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4efea} kind: Pod name: splitter-graph-4efea-5bb974bbff-2dr89 namespace: kserve-ci-e2e-test resourceVersion: "25973" uid: 7b374bd5-2f71-4887-b183-48690a08cf17 kind: Event lastTimestamp: "2026-04-24T21:50:43Z" message: Started container splitter-graph-4efea metadata: creationTimestamp: "2026-04-24T21:50:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:43Z" name: splitter-graph-4efea-5bb974bbff-2dr89.18a9696cb8b0c126 namespace: kserve-ci-e2e-test resourceVersion: "25997" uid: a525d78f-d919-4305-bbdd-fc540f923eaf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4efea} kind: Pod name: splitter-graph-4efea-5bb974bbff-2dr89 namespace: kserve-ci-e2e-test resourceVersion: "25973" uid: 7b374bd5-2f71-4887-b183-48690a08cf17 kind: Event lastTimestamp: "2026-04-24T21:50:53Z" message: Stopping container splitter-graph-4efea metadata: creationTimestamp: "2026-04-24T21:50:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:53Z" name: splitter-graph-4efea-5bb974bbff-2dr89.18a9696ef27003d6 namespace: kserve-ci-e2e-test resourceVersion: "26099" uid: 9af40d02-4d16-4ccc-a567-40219c65d95e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:50:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4efea} kind: Pod name: splitter-graph-4efea-5bb974bbff-2dr89 namespace: kserve-ci-e2e-test resourceVersion: "25973" uid: 7b374bd5-2f71-4887-b183-48690a08cf17 kind: Event lastTimestamp: "2026-04-24T21:51:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:51:19Z" name: splitter-graph-4efea-5bb974bbff-2dr89.18a9696f541ea795 namespace: kserve-ci-e2e-test resourceVersion: "26487" uid: 9ed65b74-4acb-44c1-9886-060f5915ec32 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-4efea-5bb974bbff namespace: kserve-ci-e2e-test resourceVersion: "25967" uid: ba24c8e3-2269-47c9-b3e4-28ed8da92272 kind: Event lastTimestamp: "2026-04-24T21:50:43Z" message: 'Created pod: splitter-graph-4efea-5bb974bbff-2dr89' metadata: creationTimestamp: "2026-04-24T21:50:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:43Z" name: splitter-graph-4efea-5bb974bbff.18a9696c994a2721 namespace: kserve-ci-e2e-test resourceVersion: "25972" uid: 75c0d8b8-c70e-46d8-94ce-f0cc4bdf17bb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-4efea namespace: kserve-ci-e2e-test resourceVersion: "25964" uid: d61023a1-8dd0-4b4c-a2d2-1bc43ea9239b kind: Event lastTimestamp: "2026-04-24T21:50:43Z" message: Scaled up replica set splitter-graph-4efea-5bb974bbff from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:50:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:43Z" name: splitter-graph-4efea.18a9696c98af1730 namespace: kserve-ci-e2e-test resourceVersion: "25969" uid: 084edf3b-b42e-4e59-a520-58caf79c6333 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:43Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-4efea namespace: kserve-ci-e2e-test resourceVersion: "25982" uid: b493f4c6-4d2f-45c4-acf5-fe5e0b859f53 kind: Event lastTimestamp: "2026-04-24T21:50:43Z" message: InferenceGraph [splitter-graph-4efea] is Ready metadata: creationTimestamp: "2026-04-24T21:50:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:43Z" name: splitter-graph-4efea.18a9696c9c310ea8 namespace: kserve-ci-e2e-test resourceVersion: "25984" uid: ef34780b-3cb9-442d-90dc-4e55cec98beb reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:50:53.348519Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-12404-predictor-564cf4d979-76fsj namespace: kserve-ci-e2e-test resourceVersion: "26134" uid: 5accb5ac-4323-44b4-8f66-aa7380c8d59e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-12404-predictor-564cf4d979-76fsj to ip-10-0-128-21.ec2.internal metadata: creationTimestamp: "2026-04-24T21:50:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:50:53Z" name: success-200-isvc-12404-predictor-564cf4d979-76fsj.18a9696efabf402f namespace: kserve-ci-e2e-test resourceVersion: "26140" uid: 174dad86-7b73-4e5b-8f11-2971b4bafcb8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:53Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-12404-predictor-564cf4d979-76fsj namespace: kserve-ci-e2e-test resourceVersion: "26138" uid: 5accb5ac-4323-44b4-8f66-aa7380c8d59e kind: Event lastTimestamp: "2026-04-24T21:50:53Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-12404-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:50:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:53Z" name: success-200-isvc-12404-predictor-564cf4d979-76fsj.18a9696f0528b4af namespace: kserve-ci-e2e-test resourceVersion: "26186" uid: 1c8588d0-b645-4999-9a42-bf8c32b9a853 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:54Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-12404-predictor-564cf4d979-76fsj namespace: kserve-ci-e2e-test resourceVersion: "26144" uid: 5accb5ac-4323-44b4-8f66-aa7380c8d59e kind: Event lastTimestamp: "2026-04-24T21:50:54Z" message: Add eth0 [10.132.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:50:54Z" name: success-200-isvc-12404-predictor-564cf4d979-76fsj.18a9696f38735640 namespace: kserve-ci-e2e-test resourceVersion: "26207" uid: 13985f35-1746-442a-ae5e-d8c2b7cd8075 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-12404-predictor-564cf4d979-76fsj namespace: kserve-ci-e2e-test resourceVersion: "26138" uid: 5accb5ac-4323-44b4-8f66-aa7380c8d59e kind: Event lastTimestamp: "2026-04-24T21:50:54Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1447" already present on machine metadata: creationTimestamp: "2026-04-24T21:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:54Z" name: success-200-isvc-12404-predictor-564cf4d979-76fsj.18a9696f39c50599 namespace: kserve-ci-e2e-test resourceVersion: "26210" uid: 30a66547-913f-4dd2-bceb-9f15dc2b9fba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-12404-predictor-564cf4d979-76fsj namespace: kserve-ci-e2e-test resourceVersion: "26138" uid: 5accb5ac-4323-44b4-8f66-aa7380c8d59e kind: Event lastTimestamp: "2026-04-24T21:50:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:54Z" name: success-200-isvc-12404-predictor-564cf4d979-76fsj.18a9696f3cbd8506 namespace: kserve-ci-e2e-test resourceVersion: "26212" uid: 16c2ba8b-b60c-4814-989e-ee939b4782ec reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-12404-predictor-564cf4d979-76fsj namespace: kserve-ci-e2e-test resourceVersion: "26138" uid: 5accb5ac-4323-44b4-8f66-aa7380c8d59e kind: Event lastTimestamp: "2026-04-24T21:50:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:54Z" name: success-200-isvc-12404-predictor-564cf4d979-76fsj.18a9696f3d3f8421 namespace: kserve-ci-e2e-test resourceVersion: "26213" uid: 06e228ca-75a2-4fac-9f54-b74f21702d8f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-12404-predictor-564cf4d979-76fsj namespace: kserve-ci-e2e-test resourceVersion: "26138" uid: 5accb5ac-4323-44b4-8f66-aa7380c8d59e kind: Event lastTimestamp: "2026-04-24T21:50:54Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:54Z" name: success-200-isvc-12404-predictor-564cf4d979-76fsj.18a9696f3d4d3347 namespace: kserve-ci-e2e-test resourceVersion: "26214" uid: 6c6f2228-b075-49fe-a6a3-122e568a721c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-12404-predictor-564cf4d979-76fsj namespace: kserve-ci-e2e-test resourceVersion: "26138" uid: 5accb5ac-4323-44b4-8f66-aa7380c8d59e kind: Event lastTimestamp: "2026-04-24T21:50:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:54Z" name: success-200-isvc-12404-predictor-564cf4d979-76fsj.18a9696f4080d7d8 namespace: kserve-ci-e2e-test resourceVersion: "26215" uid: 0f54c2f7-6e0e-4508-8bfe-20eda5957765 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-12404-predictor-564cf4d979-76fsj namespace: kserve-ci-e2e-test resourceVersion: "26138" uid: 5accb5ac-4323-44b4-8f66-aa7380c8d59e kind: Event lastTimestamp: "2026-04-24T21:50:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:50:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:54Z" name: success-200-isvc-12404-predictor-564cf4d979-76fsj.18a9696f40ff160c namespace: kserve-ci-e2e-test resourceVersion: "26216" uid: 8fad48c0-479a-4d5a-955b-008bf2b9f912 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-12404-predictor-564cf4d979-76fsj namespace: kserve-ci-e2e-test resourceVersion: "26138" uid: 5accb5ac-4323-44b4-8f66-aa7380c8d59e kind: Event lastTimestamp: "2026-04-24T22:00:11Z" message: 'Readiness probe failed: dial tcp 10.132.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:00:11Z" name: success-200-isvc-12404-predictor-564cf4d979-76fsj.18a9696f924a0b5f namespace: kserve-ci-e2e-test resourceVersion: "30287" uid: d8a6bd7e-55cb-44da-82f0-c83d363cdc2e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:00:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-12404-predictor-564cf4d979-76fsj namespace: kserve-ci-e2e-test resourceVersion: "26138" uid: 5accb5ac-4323-44b4-8f66-aa7380c8d59e kind: Event lastTimestamp: "2026-04-24T22:00:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:00:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:00:08Z" name: success-200-isvc-12404-predictor-564cf4d979-76fsj.18a969f02b8e3e83 namespace: kserve-ci-e2e-test resourceVersion: "30244" uid: fb903b74-6b0c-4996-97b7-599a3ea79dee reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:00:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-12404-predictor-564cf4d979-76fsj namespace: kserve-ci-e2e-test resourceVersion: "26138" uid: 5accb5ac-4323-44b4-8f66-aa7380c8d59e kind: Event lastTimestamp: "2026-04-24T22:00:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:00:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:00:08Z" name: success-200-isvc-12404-predictor-564cf4d979-76fsj.18a969f02b8eb690 namespace: kserve-ci-e2e-test resourceVersion: "30245" uid: e63aa73f-ee7b-438a-aa9c-3b2306550680 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:00:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-12404-predictor-564cf4d979-76fsj namespace: kserve-ci-e2e-test resourceVersion: "26138" uid: 5accb5ac-4323-44b4-8f66-aa7380c8d59e kind: Event lastTimestamp: "2026-04-24T22:00:11Z" message: 'Readiness probe failed: Get "https://10.132.0.46:8643/healthz": dial tcp 10.132.0.46:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:00:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:00:11Z" name: success-200-isvc-12404-predictor-564cf4d979-76fsj.18a969f106c7e9ef namespace: kserve-ci-e2e-test resourceVersion: "30286" uid: d5c53d24-8e58-4a38-bba1-aba0ed79c08f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-12404-predictor-564cf4d979 namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: dbca778c-5022-498f-b75e-a90b312ec851 kind: Event lastTimestamp: "2026-04-24T21:50:53Z" message: 'Created pod: success-200-isvc-12404-predictor-564cf4d979-76fsj' metadata: creationTimestamp: "2026-04-24T21:50:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:53Z" name: success-200-isvc-12404-predictor-564cf4d979.18a9696efa5330d3 namespace: kserve-ci-e2e-test resourceVersion: "26137" uid: 6dc6c53b-fb3d-45b2-9fc6-974a1461216e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-12404-predictor namespace: kserve-ci-e2e-test resourceVersion: "26117" uid: 7aaf1f8c-e326-4558-b2c0-e39fd2924982 kind: Event lastTimestamp: "2026-04-24T21:50:53Z" message: Scaled up replica set success-200-isvc-12404-predictor-564cf4d979 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:50:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:53Z" name: success-200-isvc-12404-predictor.18a9696ef864fc61 namespace: kserve-ci-e2e-test resourceVersion: "26124" uid: 9e20dd13-b591-4010-ad80-3e606a3370d0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:51:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-12404-predictor namespace: kserve-ci-e2e-test resourceVersion: "26119" uid: f3577b16-366c-4dbf-860a-bdab1bbf0687 kind: Event lastTimestamp: "2026-04-24T21:51:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:51:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:51:23Z" name: success-200-isvc-12404-predictor.18a9697277095d75 namespace: kserve-ci-e2e-test resourceVersion: "26523" uid: 1158a778-767e-47cd-a709-e54e1c4da752 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:51:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-12404-predictor namespace: kserve-ci-e2e-test resourceVersion: "26119" uid: f3577b16-366c-4dbf-860a-bdab1bbf0687 kind: Event lastTimestamp: "2026-04-24T21:51:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:51:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:51:23Z" name: success-200-isvc-12404-predictor.18a96972776abd95 namespace: kserve-ci-e2e-test resourceVersion: "26524" uid: 92ea9bba-1170-4560-8f57-0ccdd7df2ca2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:51:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-12404-predictor namespace: kserve-ci-e2e-test resourceVersion: "26362" uid: f3577b16-366c-4dbf-860a-bdab1bbf0687 kind: Event lastTimestamp: "2026-04-24T21:51:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:51:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:51:53Z" name: success-200-isvc-12404-predictor.18a9697974e12095 namespace: kserve-ci-e2e-test resourceVersion: "26776" uid: d327327e-914f-488e-b43c-ca9de82c659c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:51:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-12404-predictor namespace: kserve-ci-e2e-test resourceVersion: "26362" uid: f3577b16-366c-4dbf-860a-bdab1bbf0687 kind: Event lastTimestamp: "2026-04-24T21:51:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:51:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:51:53Z" name: success-200-isvc-12404-predictor.18a96979754c5629 namespace: kserve-ci-e2e-test resourceVersion: "26779" uid: 839ee34e-f46b-4aeb-ae2f-a47266d7678d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:50:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-12404 namespace: kserve-ci-e2e-test resourceVersion: "26102" uid: 37779722-1a8e-4290-b454-af0403139c07 kind: Event lastTimestamp: "2026-04-24T21:51:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:50:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:51:42Z" name: success-200-isvc-12404.18a9696ef86ddd0a namespace: kserve-ci-e2e-test resourceVersion: "26691" uid: da7e8fd9-38ec-4156-813b-bc90f3127bc4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:51:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-12404 namespace: kserve-ci-e2e-test resourceVersion: "26689" uid: 37779722-1a8e-4290-b454-af0403139c07 kind: Event lastTimestamp: "2026-04-24T21:51:41Z" message: InferenceService [success-200-isvc-12404] is Ready metadata: creationTimestamp: "2026-04-24T21:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:51:41Z" name: success-200-isvc-12404.18a9697a4d1d238f namespace: kserve-ci-e2e-test resourceVersion: "26690" uid: 5cd1b7c6-4508-45c1-99db-5d00af10d15b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:30:39.006298Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3a238-predictor-644f78f4dc-brf2z namespace: kserve-ci-e2e-test resourceVersion: "15826" uid: ef252636-56e1-4ba4-8b22-135c16a6121b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3a238-predictor-644f78f4dc-brf2z to ip-10-0-128-21.ec2.internal metadata: creationTimestamp: "2026-04-24T21:30:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:30:39Z" name: success-200-isvc-3a238-predictor-644f78f4dc-brf2z.18a968543e4f726f namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: d6cc126d-f580-436c-af7c-bad892b6982b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:39Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3a238-predictor-644f78f4dc-brf2z namespace: kserve-ci-e2e-test resourceVersion: "15832" uid: ef252636-56e1-4ba4-8b22-135c16a6121b kind: Event lastTimestamp: "2026-04-24T21:30:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-3a238-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:30:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:39Z" name: success-200-isvc-3a238-predictor-644f78f4dc-brf2z.18a968544aaf2ddb namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: bfd7de0c-e97d-4500-a17f-366a982ac1bb reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:40Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3a238-predictor-644f78f4dc-brf2z namespace: kserve-ci-e2e-test resourceVersion: "15845" uid: ef252636-56e1-4ba4-8b22-135c16a6121b kind: Event lastTimestamp: "2026-04-24T21:30:40Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:30:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:30:40Z" name: success-200-isvc-3a238-predictor-644f78f4dc-brf2z.18a968547b1f7caf namespace: kserve-ci-e2e-test resourceVersion: "15890" uid: f65e72b0-f3e4-438b-b314-c74b06bea531 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3a238-predictor-644f78f4dc-brf2z namespace: kserve-ci-e2e-test resourceVersion: "15832" uid: ef252636-56e1-4ba4-8b22-135c16a6121b kind: Event lastTimestamp: "2026-04-24T21:30:40Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1447" already present on machine metadata: creationTimestamp: "2026-04-24T21:30:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:40Z" name: success-200-isvc-3a238-predictor-644f78f4dc-brf2z.18a968547c75f286 namespace: kserve-ci-e2e-test resourceVersion: "15892" uid: b024a7ce-9330-4837-91a9-fd30fdf060de reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3a238-predictor-644f78f4dc-brf2z namespace: kserve-ci-e2e-test resourceVersion: "15832" uid: ef252636-56e1-4ba4-8b22-135c16a6121b kind: Event lastTimestamp: "2026-04-24T21:30:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:30:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:40Z" name: success-200-isvc-3a238-predictor-644f78f4dc-brf2z.18a968547f67aed7 namespace: kserve-ci-e2e-test resourceVersion: "15894" uid: 8f2a0453-4546-4bc3-8b8e-22cfa94817f3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3a238-predictor-644f78f4dc-brf2z namespace: kserve-ci-e2e-test resourceVersion: "15832" uid: ef252636-56e1-4ba4-8b22-135c16a6121b kind: Event lastTimestamp: "2026-04-24T21:30:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:30:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:40Z" name: success-200-isvc-3a238-predictor-644f78f4dc-brf2z.18a968547fd86cae namespace: kserve-ci-e2e-test resourceVersion: "15895" uid: 4166d1d2-7d30-4758-a177-02df38e9f17a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3a238-predictor-644f78f4dc-brf2z namespace: kserve-ci-e2e-test resourceVersion: "15832" uid: ef252636-56e1-4ba4-8b22-135c16a6121b kind: Event lastTimestamp: "2026-04-24T21:30:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:30:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:40Z" name: success-200-isvc-3a238-predictor-644f78f4dc-brf2z.18a968547fe65c6f namespace: kserve-ci-e2e-test resourceVersion: "15896" uid: 43ad8bbd-21c0-4809-a94f-8cbb035a1708 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3a238-predictor-644f78f4dc-brf2z namespace: kserve-ci-e2e-test resourceVersion: "15832" uid: ef252636-56e1-4ba4-8b22-135c16a6121b kind: Event lastTimestamp: "2026-04-24T21:30:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:30:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:40Z" name: success-200-isvc-3a238-predictor-644f78f4dc-brf2z.18a9685483af6559 namespace: kserve-ci-e2e-test resourceVersion: "15901" uid: 65bb87e0-d540-47c9-960f-af4d47719e2b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3a238-predictor-644f78f4dc-brf2z namespace: kserve-ci-e2e-test resourceVersion: "15832" uid: ef252636-56e1-4ba4-8b22-135c16a6121b kind: Event lastTimestamp: "2026-04-24T21:30:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:30:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:40Z" name: success-200-isvc-3a238-predictor-644f78f4dc-brf2z.18a9685484645025 namespace: kserve-ci-e2e-test resourceVersion: "15902" uid: 321ea351-86d6-429b-bc53-a808f2d8393a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:30:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3a238-predictor-644f78f4dc-brf2z namespace: kserve-ci-e2e-test resourceVersion: "15832" uid: ef252636-56e1-4ba4-8b22-135c16a6121b kind: Event lastTimestamp: "2026-04-24T21:39:56Z" message: 'Readiness probe failed: dial tcp 10.132.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:30:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:56Z" name: success-200-isvc-3a238-predictor-644f78f4dc-brf2z.18a96854add4295f namespace: kserve-ci-e2e-test resourceVersion: "20238" uid: c8e75c10-294b-49ff-990f-4017549ca91f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3a238-predictor-644f78f4dc-brf2z namespace: kserve-ci-e2e-test resourceVersion: "15832" uid: ef252636-56e1-4ba4-8b22-135c16a6121b kind: Event lastTimestamp: "2026-04-24T21:39:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:39:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:53Z" name: success-200-isvc-3a238-predictor-644f78f4dc-brf2z.18a968d56fd60830 namespace: kserve-ci-e2e-test resourceVersion: "20127" uid: ae88d2c0-4b76-48f0-a266-41e20019900d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3a238-predictor-644f78f4dc-brf2z namespace: kserve-ci-e2e-test resourceVersion: "15832" uid: ef252636-56e1-4ba4-8b22-135c16a6121b kind: Event lastTimestamp: "2026-04-24T21:39:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:39:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:53Z" name: success-200-isvc-3a238-predictor-644f78f4dc-brf2z.18a968d56fd6a7a1 namespace: kserve-ci-e2e-test resourceVersion: "20129" uid: a5cd46f4-67d5-40eb-90ed-5df0099781c9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3a238-predictor-644f78f4dc-brf2z namespace: kserve-ci-e2e-test resourceVersion: "15832" uid: ef252636-56e1-4ba4-8b22-135c16a6121b kind: Event lastTimestamp: "2026-04-24T21:39:56Z" message: 'Readiness probe failed: Get "https://10.132.0.32:8643/healthz": dial tcp 10.132.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:39:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:56Z" name: success-200-isvc-3a238-predictor-644f78f4dc-brf2z.18a968d62231ac0e namespace: kserve-ci-e2e-test resourceVersion: "20237" uid: 545564fc-67e9-4d6e-9d3a-a5f58a32fcd2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3a238-predictor-644f78f4dc namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: 999e58cc-06ee-41f3-8ae7-770f0d8bd9f5 kind: Event lastTimestamp: "2026-04-24T21:30:38Z" message: 'Created pod: success-200-isvc-3a238-predictor-644f78f4dc-brf2z' metadata: creationTimestamp: "2026-04-24T21:30:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:39Z" name: success-200-isvc-3a238-predictor-644f78f4dc.18a968543de9e96e namespace: kserve-ci-e2e-test resourceVersion: "15831" uid: 0486e3e8-fb42-4c88-b84b-35a8822ae318 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3a238-predictor namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: ed6e0e86-0a8f-46af-b36b-a03f669aabd0 kind: Event lastTimestamp: "2026-04-24T21:30:38Z" message: Scaled up replica set success-200-isvc-3a238-predictor-644f78f4dc from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:30:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:38Z" name: success-200-isvc-3a238-predictor.18a968543c0cdb6d namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: a0a0d6a1-abb2-4b71-a87e-6ccd927d63dc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:30:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3a238-predictor namespace: kserve-ci-e2e-test resourceVersion: "15807" uid: 21339587-f7d3-4b1c-9e43-3c8bf1ebd231 kind: Event lastTimestamp: "2026-04-24T21:31:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:30:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:31:08Z" name: success-200-isvc-3a238-predictor.18a96857ba84809d namespace: kserve-ci-e2e-test resourceVersion: "16218" uid: 786c1c95-fa6f-4c64-b19e-e3052e41131d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:30:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3a238-predictor namespace: kserve-ci-e2e-test resourceVersion: "15807" uid: 21339587-f7d3-4b1c-9e43-3c8bf1ebd231 kind: Event lastTimestamp: "2026-04-24T21:31:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:30:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:31:08Z" name: success-200-isvc-3a238-predictor.18a96857bad0d6f5 namespace: kserve-ci-e2e-test resourceVersion: "16219" uid: e9e901e1-b5b1-4bf6-b5e2-984664bf1f03 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:31:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3a238-predictor namespace: kserve-ci-e2e-test resourceVersion: "16061" uid: 21339587-f7d3-4b1c-9e43-3c8bf1ebd231 kind: Event lastTimestamp: "2026-04-24T21:31:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:31:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:31:39Z" name: success-200-isvc-3a238-predictor.18a9685eb8442ae9 namespace: kserve-ci-e2e-test resourceVersion: "16495" uid: bf85ecdf-519c-4b08-b80c-f217a9535d38 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:31:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3a238-predictor namespace: kserve-ci-e2e-test resourceVersion: "16061" uid: 21339587-f7d3-4b1c-9e43-3c8bf1ebd231 kind: Event lastTimestamp: "2026-04-24T21:31:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:31:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:31:39Z" name: success-200-isvc-3a238-predictor.18a9685eb8a8d69f namespace: kserve-ci-e2e-test resourceVersion: "16497" uid: e82c6c4e-b78e-4f1b-bc4d-5755e862a60f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:30:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3a238 namespace: kserve-ci-e2e-test resourceVersion: "15776" uid: 7d7cd86c-afaa-4b60-bffe-c6c18fc02651 kind: Event lastTimestamp: "2026-04-24T21:31:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:30:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:31:26Z" name: success-200-isvc-3a238.18a968543bea10f2 namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: 097d5d5b-fe52-4f69-be13-3e27cb728040 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:31:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3a238 namespace: kserve-ci-e2e-test resourceVersion: "16377" uid: 7d7cd86c-afaa-4b60-bffe-c6c18fc02651 kind: Event lastTimestamp: "2026-04-24T21:31:26Z" message: InferenceService [success-200-isvc-3a238] is Ready metadata: creationTimestamp: "2026-04-24T21:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:31:26Z" name: success-200-isvc-3a238.18a9685f687a77d5 namespace: kserve-ci-e2e-test resourceVersion: "16378" uid: f2f01398-6192-4458-af5f-35298859ae43 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:49:43.102867Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4efea-predictor-6bd955dcc9-9tf9f namespace: kserve-ci-e2e-test resourceVersion: "24965" uid: b77acea4-fae8-402c-b34d-8ff4efaa4d78 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4efea-predictor-6bd955dcc9-9tf9f to ip-10-0-128-21.ec2.internal metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:49:43Z" name: success-200-isvc-4efea-predictor-6bd955dcc9-9tf9f.18a9695e9fc7a77c namespace: kserve-ci-e2e-test resourceVersion: "24970" uid: a669ba5a-e0a6-45f1-a048-d17435d455ad reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4efea-predictor-6bd955dcc9-9tf9f namespace: kserve-ci-e2e-test resourceVersion: "24980" uid: b77acea4-fae8-402c-b34d-8ff4efaa4d78 kind: Event lastTimestamp: "2026-04-24T21:49:43Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:49:43Z" name: success-200-isvc-4efea-predictor-6bd955dcc9-9tf9f.18a9695eb92ab4a9 namespace: kserve-ci-e2e-test resourceVersion: "25009" uid: 01fed2cd-2995-4472-8a4e-6e85c68fb617 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4efea-predictor-6bd955dcc9-9tf9f namespace: kserve-ci-e2e-test resourceVersion: "24967" uid: b77acea4-fae8-402c-b34d-8ff4efaa4d78 kind: Event lastTimestamp: "2026-04-24T21:49:43Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1447" already present on machine metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:43Z" name: success-200-isvc-4efea-predictor-6bd955dcc9-9tf9f.18a9695eba992119 namespace: kserve-ci-e2e-test resourceVersion: "25012" uid: fbeaf734-bfd5-4368-a9a1-750616a9ef8b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4efea-predictor-6bd955dcc9-9tf9f namespace: kserve-ci-e2e-test resourceVersion: "24967" uid: b77acea4-fae8-402c-b34d-8ff4efaa4d78 kind: Event lastTimestamp: "2026-04-24T21:49:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:43Z" name: success-200-isvc-4efea-predictor-6bd955dcc9-9tf9f.18a9695ebd93ae17 namespace: kserve-ci-e2e-test resourceVersion: "25013" uid: eac58050-fbdd-4105-8419-2a6c2d632d53 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4efea-predictor-6bd955dcc9-9tf9f namespace: kserve-ci-e2e-test resourceVersion: "24967" uid: b77acea4-fae8-402c-b34d-8ff4efaa4d78 kind: Event lastTimestamp: "2026-04-24T21:49:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:43Z" name: success-200-isvc-4efea-predictor-6bd955dcc9-9tf9f.18a9695ebe1b1f09 namespace: kserve-ci-e2e-test resourceVersion: "25015" uid: 2237eb6a-cb62-4437-890c-89645f112831 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4efea-predictor-6bd955dcc9-9tf9f namespace: kserve-ci-e2e-test resourceVersion: "24967" uid: b77acea4-fae8-402c-b34d-8ff4efaa4d78 kind: Event lastTimestamp: "2026-04-24T21:49:43Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:43Z" name: success-200-isvc-4efea-predictor-6bd955dcc9-9tf9f.18a9695ebe27d763 namespace: kserve-ci-e2e-test resourceVersion: "25017" uid: c9e689a3-79a1-49c2-9f60-8bb9877b8185 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4efea-predictor-6bd955dcc9-9tf9f namespace: kserve-ci-e2e-test resourceVersion: "24967" uid: b77acea4-fae8-402c-b34d-8ff4efaa4d78 kind: Event lastTimestamp: "2026-04-24T21:49:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:43Z" name: success-200-isvc-4efea-predictor-6bd955dcc9-9tf9f.18a9695ec0ddbdbf namespace: kserve-ci-e2e-test resourceVersion: "25021" uid: 8f9ed6d1-3c2d-45ed-ab41-7b621552ac90 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4efea-predictor-6bd955dcc9-9tf9f namespace: kserve-ci-e2e-test resourceVersion: "24967" uid: b77acea4-fae8-402c-b34d-8ff4efaa4d78 kind: Event lastTimestamp: "2026-04-24T21:49:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:43Z" name: success-200-isvc-4efea-predictor-6bd955dcc9-9tf9f.18a9695ec15314ff namespace: kserve-ci-e2e-test resourceVersion: "25022" uid: 966d3f33-c7ea-4e90-ac42-56b5cf92347e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:49:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4efea-predictor-6bd955dcc9-9tf9f namespace: kserve-ci-e2e-test resourceVersion: "24967" uid: b77acea4-fae8-402c-b34d-8ff4efaa4d78 kind: Event lastTimestamp: "2026-04-24T21:50:21Z" message: 'Readiness probe failed: dial tcp 10.132.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:21Z" name: success-200-isvc-4efea-predictor-6bd955dcc9-9tf9f.18a9695f37157af5 namespace: kserve-ci-e2e-test resourceVersion: "25759" uid: 91c7db64-5dd5-4870-bf30-650d3649b7e0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4efea-predictor-6bd955dcc9-9tf9f namespace: kserve-ci-e2e-test resourceVersion: "24967" uid: b77acea4-fae8-402c-b34d-8ff4efaa4d78 kind: Event lastTimestamp: "2026-04-24T21:50:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:50:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:53Z" name: success-200-isvc-4efea-predictor-6bd955dcc9-9tf9f.18a9696ef98e7dab namespace: kserve-ci-e2e-test resourceVersion: "26132" uid: 8db8bad8-28b2-45f6-a57e-8df4a5798955 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4efea-predictor-6bd955dcc9-9tf9f namespace: kserve-ci-e2e-test resourceVersion: "24967" uid: b77acea4-fae8-402c-b34d-8ff4efaa4d78 kind: Event lastTimestamp: "2026-04-24T21:50:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:50:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:53Z" name: success-200-isvc-4efea-predictor-6bd955dcc9-9tf9f.18a9696ef98ee6a4 namespace: kserve-ci-e2e-test resourceVersion: "26133" uid: 4b238bd8-5244-453d-9a23-137eab41e8a8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4efea-predictor-6bd955dcc9-9tf9f namespace: kserve-ci-e2e-test resourceVersion: "24967" uid: b77acea4-fae8-402c-b34d-8ff4efaa4d78 kind: Event lastTimestamp: "2026-04-24T21:50:56Z" message: 'Readiness probe failed: Get "https://10.132.0.43:8643/healthz": dial tcp 10.132.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:56Z" name: success-200-isvc-4efea-predictor-6bd955dcc9-9tf9f.18a9696fbf409062 namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 54d1f301-c8fd-4396-8d52-cc94714ce063 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4efea-predictor-6bd955dcc9 namespace: kserve-ci-e2e-test resourceVersion: "24941" uid: 9a1036ab-b7eb-4979-921a-1938f71b9582 kind: Event lastTimestamp: "2026-04-24T21:49:43Z" message: 'Created pod: success-200-isvc-4efea-predictor-6bd955dcc9-9tf9f' metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:49:43Z" name: success-200-isvc-4efea-predictor-6bd955dcc9.18a9695e9f3d4c57 namespace: kserve-ci-e2e-test resourceVersion: "24966" uid: f00b15f7-e213-4fc1-9c08-5268dc849dd1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4efea-predictor namespace: kserve-ci-e2e-test resourceVersion: "24937" uid: 79e62ae9-9ff9-448a-92ea-2bcff767d17b kind: Event lastTimestamp: "2026-04-24T21:49:43Z" message: Scaled up replica set success-200-isvc-4efea-predictor-6bd955dcc9 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:49:43Z" name: success-200-isvc-4efea-predictor.18a9695e9b75b204 namespace: kserve-ci-e2e-test resourceVersion: "24943" uid: b6d92fb0-91c4-4e5e-83b8-181753358434 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:49:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4efea-predictor namespace: kserve-ci-e2e-test resourceVersion: "24940" uid: c92f8fa0-5fa4-4cab-a6ca-237623423969 kind: Event lastTimestamp: "2026-04-24T21:50:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:49:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:28Z" name: success-200-isvc-4efea-predictor.18a969621ac74cec namespace: kserve-ci-e2e-test resourceVersion: "25824" uid: 92883548-a6d1-4473-992c-a4c556a18688 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:49:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4efea-predictor namespace: kserve-ci-e2e-test resourceVersion: "24940" uid: c92f8fa0-5fa4-4cab-a6ca-237623423969 kind: Event lastTimestamp: "2026-04-24T21:50:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:49:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:28Z" name: success-200-isvc-4efea-predictor.18a969621b29ffe9 namespace: kserve-ci-e2e-test resourceVersion: "25825" uid: 02d76d32-bea4-4a5b-a907-218a340a4e06 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4efea-predictor namespace: kserve-ci-e2e-test resourceVersion: "25440" uid: c92f8fa0-5fa4-4cab-a6ca-237623423969 kind: Event lastTimestamp: "2026-04-24T21:50:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:50:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:43Z" name: success-200-isvc-4efea-predictor.18a9696c97b1eb7f namespace: kserve-ci-e2e-test resourceVersion: "25965" uid: 4d015bf2-32a9-4b9b-870e-991bc20be520 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4efea-predictor namespace: kserve-ci-e2e-test resourceVersion: "25440" uid: c92f8fa0-5fa4-4cab-a6ca-237623423969 kind: Event lastTimestamp: "2026-04-24T21:50:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:50:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:43Z" name: success-200-isvc-4efea-predictor.18a9696c9836da82 namespace: kserve-ci-e2e-test resourceVersion: "25968" uid: 4bd3fd9e-8fc1-42f0-99a5-644c77a551e6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4efea namespace: kserve-ci-e2e-test resourceVersion: "24916" uid: 42b69995-869f-4560-a890-1137301ad92f kind: Event lastTimestamp: "2026-04-24T21:50:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:31Z" name: success-200-isvc-4efea.18a9695e9b515d47 namespace: kserve-ci-e2e-test resourceVersion: "25868" uid: e7559935-ab16-442c-a1c7-3c5ded44f77d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4efea namespace: kserve-ci-e2e-test resourceVersion: "24916" uid: 42b69995-869f-4560-a890-1137301ad92f kind: Event lastTimestamp: "2026-04-24T21:49:43Z" message: 'Failed to update status for InferenceService "success-200-isvc-4efea": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4efea": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:49:43Z" name: success-200-isvc-4efea.18a9695e9b9e80f2 namespace: kserve-ci-e2e-test resourceVersion: "24945" uid: 1a42a6f2-0e91-4188-b9c1-3246ba88dd17 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4efea namespace: kserve-ci-e2e-test resourceVersion: "24916" uid: 42b69995-869f-4560-a890-1137301ad92f kind: Event lastTimestamp: "2026-04-24T21:49:43Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4efea": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:49:43Z" name: success-200-isvc-4efea.18a9695e9b9ed184 namespace: kserve-ci-e2e-test resourceVersion: "24947" uid: 18ebc398-c5fd-4db0-bf19-7d2b4a988484 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4efea namespace: kserve-ci-e2e-test resourceVersion: "25866" uid: 42b69995-869f-4560-a890-1137301ad92f kind: Event lastTimestamp: "2026-04-24T21:50:31Z" message: InferenceService [success-200-isvc-4efea] is Ready metadata: creationTimestamp: "2026-04-24T21:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:31Z" name: success-200-isvc-4efea.18a96969f189e214 namespace: kserve-ci-e2e-test resourceVersion: "25867" uid: f62bf626-070b-4a43-8b1b-05b2a67d6fde reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:41:04.133081Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-64806-predictor-6bdc69944f-rmdz9 namespace: kserve-ci-e2e-test resourceVersion: "21114" uid: e66a7a28-134b-4266-81b7-b0eb7ef91ae9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-64806-predictor-6bdc69944f-rmdz9 to ip-10-0-128-21.ec2.internal metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:41:04Z" name: success-200-isvc-64806-predictor-6bdc69944f-rmdz9.18a968e5cac51ad4 namespace: kserve-ci-e2e-test resourceVersion: "21119" uid: eabe5ec4-4bd8-4d4c-ac56-58bd80e2b87b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-64806-predictor-6bdc69944f-rmdz9 namespace: kserve-ci-e2e-test resourceVersion: "21124" uid: e66a7a28-134b-4266-81b7-b0eb7ef91ae9 kind: Event lastTimestamp: "2026-04-24T21:41:04Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:41:04Z" name: success-200-isvc-64806-predictor-6bdc69944f-rmdz9.18a968e5e4159407 namespace: kserve-ci-e2e-test resourceVersion: "21157" uid: e4a6ea7d-9428-4978-99a5-8f36e61bc037 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-64806-predictor-6bdc69944f-rmdz9 namespace: kserve-ci-e2e-test resourceVersion: "21116" uid: e66a7a28-134b-4266-81b7-b0eb7ef91ae9 kind: Event lastTimestamp: "2026-04-24T21:41:04Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1447" already present on machine metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:04Z" name: success-200-isvc-64806-predictor-6bdc69944f-rmdz9.18a968e5e54eee62 namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: bf521d17-8310-4ae5-98c8-2be592ba0d3f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-64806-predictor-6bdc69944f-rmdz9 namespace: kserve-ci-e2e-test resourceVersion: "21116" uid: e66a7a28-134b-4266-81b7-b0eb7ef91ae9 kind: Event lastTimestamp: "2026-04-24T21:41:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:04Z" name: success-200-isvc-64806-predictor-6bdc69944f-rmdz9.18a968e5e8279c36 namespace: kserve-ci-e2e-test resourceVersion: "21161" uid: 26519cb3-8b9e-405c-ac96-19e023ef33d5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-64806-predictor-6bdc69944f-rmdz9 namespace: kserve-ci-e2e-test resourceVersion: "21116" uid: e66a7a28-134b-4266-81b7-b0eb7ef91ae9 kind: Event lastTimestamp: "2026-04-24T21:41:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:04Z" name: success-200-isvc-64806-predictor-6bdc69944f-rmdz9.18a968e5e89b9e9a namespace: kserve-ci-e2e-test resourceVersion: "21162" uid: 00e60c30-718c-4703-baa3-a096e59bd42d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-64806-predictor-6bdc69944f-rmdz9 namespace: kserve-ci-e2e-test resourceVersion: "21116" uid: e66a7a28-134b-4266-81b7-b0eb7ef91ae9 kind: Event lastTimestamp: "2026-04-24T21:41:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:04Z" name: success-200-isvc-64806-predictor-6bdc69944f-rmdz9.18a968e5e8a77530 namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: fa0a9555-4b55-4558-ae8d-de1c9d686e3f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-64806-predictor-6bdc69944f-rmdz9 namespace: kserve-ci-e2e-test resourceVersion: "21116" uid: e66a7a28-134b-4266-81b7-b0eb7ef91ae9 kind: Event lastTimestamp: "2026-04-24T21:41:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:04Z" name: success-200-isvc-64806-predictor-6bdc69944f-rmdz9.18a968e5eb4160c2 namespace: kserve-ci-e2e-test resourceVersion: "21166" uid: 21877b62-9060-466e-9191-be8bb254e8ee reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-64806-predictor-6bdc69944f-rmdz9 namespace: kserve-ci-e2e-test resourceVersion: "21116" uid: e66a7a28-134b-4266-81b7-b0eb7ef91ae9 kind: Event lastTimestamp: "2026-04-24T21:41:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:04Z" name: success-200-isvc-64806-predictor-6bdc69944f-rmdz9.18a968e5ebafe882 namespace: kserve-ci-e2e-test resourceVersion: "21167" uid: 0f2fa24a-1bd4-4ac4-8cb9-d471d718abac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:41:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-64806-predictor-6bdc69944f-rmdz9 namespace: kserve-ci-e2e-test resourceVersion: "21116" uid: e66a7a28-134b-4266-81b7-b0eb7ef91ae9 kind: Event lastTimestamp: "2026-04-24T21:50:21Z" message: 'Readiness probe failed: dial tcp 10.132.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:41:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:21Z" name: success-200-isvc-64806-predictor-6bdc69944f-rmdz9.18a968e636fea32f namespace: kserve-ci-e2e-test resourceVersion: "25764" uid: 803e4f10-d4e7-43f4-b6d5-b590b1817c96 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-64806-predictor-6bdc69944f-rmdz9 namespace: kserve-ci-e2e-test resourceVersion: "21116" uid: e66a7a28-134b-4266-81b7-b0eb7ef91ae9 kind: Event lastTimestamp: "2026-04-24T21:50:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:50:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:18Z" name: success-200-isvc-64806-predictor-6bdc69944f-rmdz9.18a96966fb18b596 namespace: kserve-ci-e2e-test resourceVersion: "25654" uid: ea455c8d-608f-4f0e-a2fb-7bcfd13abeb2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-64806-predictor-6bdc69944f-rmdz9 namespace: kserve-ci-e2e-test resourceVersion: "21116" uid: e66a7a28-134b-4266-81b7-b0eb7ef91ae9 kind: Event lastTimestamp: "2026-04-24T21:50:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:50:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:19Z" name: success-200-isvc-64806-predictor-6bdc69944f-rmdz9.18a96966fb198163 namespace: kserve-ci-e2e-test resourceVersion: "25655" uid: f94a02aa-7468-4ed8-a5e5-50eb58701949 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-64806-predictor-6bdc69944f-rmdz9 namespace: kserve-ci-e2e-test resourceVersion: "21116" uid: e66a7a28-134b-4266-81b7-b0eb7ef91ae9 kind: Event lastTimestamp: "2026-04-24T21:50:21Z" message: 'Readiness probe failed: Get "https://10.132.0.40:8643/healthz": dial tcp 10.132.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:50:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:21Z" name: success-200-isvc-64806-predictor-6bdc69944f-rmdz9.18a96967ab4fd177 namespace: kserve-ci-e2e-test resourceVersion: "25763" uid: c0f9a4c0-4f6b-4db4-a39c-c032f02e70f6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-64806-predictor-6bdc69944f namespace: kserve-ci-e2e-test resourceVersion: "21098" uid: 4e17ba54-7177-4299-80da-c7fb83b2c726 kind: Event lastTimestamp: "2026-04-24T21:41:04Z" message: 'Created pod: success-200-isvc-64806-predictor-6bdc69944f-rmdz9' metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:04Z" name: success-200-isvc-64806-predictor-6bdc69944f.18a968e5ca6573c6 namespace: kserve-ci-e2e-test resourceVersion: "21115" uid: a9097f5a-beb8-4e27-90a5-61a38055fcf8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-64806-predictor namespace: kserve-ci-e2e-test resourceVersion: "21092" uid: beb20933-e67f-4216-b285-dbcbad5969bc kind: Event lastTimestamp: "2026-04-24T21:41:04Z" message: Scaled up replica set success-200-isvc-64806-predictor-6bdc69944f from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:04Z" name: success-200-isvc-64806-predictor.18a968e5c83219f1 namespace: kserve-ci-e2e-test resourceVersion: "21103" uid: 2aa9f318-b043-4a7a-9d93-4e72362d0f61 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:41:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-64806-predictor namespace: kserve-ci-e2e-test resourceVersion: "21097" uid: 8c572071-4b34-4d2e-9b69-a5515f32071e kind: Event lastTimestamp: "2026-04-24T21:41:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:34Z" name: success-200-isvc-64806-predictor.18a968e946c5d79f namespace: kserve-ci-e2e-test resourceVersion: "21481" uid: 7996b6ae-3aeb-4022-99f1-4de7dcfc9f2b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:41:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-64806-predictor namespace: kserve-ci-e2e-test resourceVersion: "21097" uid: 8c572071-4b34-4d2e-9b69-a5515f32071e kind: Event lastTimestamp: "2026-04-24T21:41:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:34Z" name: success-200-isvc-64806-predictor.18a968e9472a589b namespace: kserve-ci-e2e-test resourceVersion: "21482" uid: 7877f91d-d3b2-454d-94ae-2169d990f7fe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:41:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-64806-predictor namespace: kserve-ci-e2e-test resourceVersion: "21325" uid: 8c572071-4b34-4d2e-9b69-a5515f32071e kind: Event lastTimestamp: "2026-04-24T21:42:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:41:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:42:04Z" name: success-200-isvc-64806-predictor.18a968f04605a29a namespace: kserve-ci-e2e-test resourceVersion: "21747" uid: 5385b55b-7b00-4cf3-88ce-e9abbc854c14 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:41:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-64806-predictor namespace: kserve-ci-e2e-test resourceVersion: "21325" uid: 8c572071-4b34-4d2e-9b69-a5515f32071e kind: Event lastTimestamp: "2026-04-24T21:42:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:41:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:42:04Z" name: success-200-isvc-64806-predictor.18a968f0466c5782 namespace: kserve-ci-e2e-test resourceVersion: "21752" uid: 51461fbb-cfc1-4daf-b994-25197f040538 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-64806 namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: 83d92382-5522-46cf-a7a9-871c4c259678 kind: Event lastTimestamp: "2026-04-24T21:41:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:41:52Z" name: success-200-isvc-64806.18a968e5c826e0bf namespace: kserve-ci-e2e-test resourceVersion: "21640" uid: e5054780-ed7c-496e-937b-ed8317b10c46 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-64806 namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: 83d92382-5522-46cf-a7a9-871c4c259678 kind: Event lastTimestamp: "2026-04-24T21:41:04Z" message: 'Failed to update status for InferenceService "success-200-isvc-64806": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-64806": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:41:04Z" name: success-200-isvc-64806.18a968e5c8c8ed15 namespace: kserve-ci-e2e-test resourceVersion: "21107" uid: 29b6d4bc-d093-4737-a8a9-3805874ce4cf reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-64806 namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: 83d92382-5522-46cf-a7a9-871c4c259678 kind: Event lastTimestamp: "2026-04-24T21:41:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-64806": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:41:04Z" name: success-200-isvc-64806.18a968e5c8c95565 namespace: kserve-ci-e2e-test resourceVersion: "21109" uid: b5efca39-d358-4533-8208-95cf37375484 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-64806 namespace: kserve-ci-e2e-test resourceVersion: "21637" uid: 83d92382-5522-46cf-a7a9-871c4c259678 kind: Event lastTimestamp: "2026-04-24T21:41:52Z" message: InferenceService [success-200-isvc-64806] is Ready metadata: creationTimestamp: "2026-04-24T21:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:41:52Z" name: success-200-isvc-64806.18a968f0f1a64fd1 namespace: kserve-ci-e2e-test resourceVersion: "21638" uid: 21c7152b-308c-48b6-b431-ec2c9e60dc28 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:40:28.191615Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-75b0d-predictor-9549449fc-92fhr namespace: kserve-ci-e2e-test resourceVersion: "20612" uid: d6580f8e-1f40-4415-b2d8-36099f4708a8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-75b0d-predictor-9549449fc-92fhr to ip-10-0-128-21.ec2.internal metadata: creationTimestamp: "2026-04-24T21:40:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:40:28Z" name: success-200-isvc-75b0d-predictor-9549449fc-92fhr.18a968dd6c7de732 namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: b3e80e0a-b6d4-4c68-b3b1-314e2233851a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:28Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-75b0d-predictor-9549449fc-92fhr namespace: kserve-ci-e2e-test resourceVersion: "20614" uid: d6580f8e-1f40-4415-b2d8-36099f4708a8 kind: Event lastTimestamp: "2026-04-24T21:40:28Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-75b0d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:28Z" name: success-200-isvc-75b0d-predictor-9549449fc-92fhr.18a968dd795c88dc namespace: kserve-ci-e2e-test resourceVersion: "20664" uid: 6f5fda0d-819b-445d-9084-a7ec2b32e0df reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:29Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-75b0d-predictor-9549449fc-92fhr namespace: kserve-ci-e2e-test resourceVersion: "20620" uid: d6580f8e-1f40-4415-b2d8-36099f4708a8 kind: Event lastTimestamp: "2026-04-24T21:40:29Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:40:29Z" name: success-200-isvc-75b0d-predictor-9549449fc-92fhr.18a968ddaa5472fd namespace: kserve-ci-e2e-test resourceVersion: "20688" uid: 06ef59cb-359c-4fa1-8b8f-0578ae428db1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-75b0d-predictor-9549449fc-92fhr namespace: kserve-ci-e2e-test resourceVersion: "20614" uid: d6580f8e-1f40-4415-b2d8-36099f4708a8 kind: Event lastTimestamp: "2026-04-24T21:40:29Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1447" already present on machine metadata: creationTimestamp: "2026-04-24T21:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:29Z" name: success-200-isvc-75b0d-predictor-9549449fc-92fhr.18a968ddabc3aae8 namespace: kserve-ci-e2e-test resourceVersion: "20691" uid: 84a912b5-003b-4e1c-884b-c06369c9e69b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-75b0d-predictor-9549449fc-92fhr namespace: kserve-ci-e2e-test resourceVersion: "20614" uid: d6580f8e-1f40-4415-b2d8-36099f4708a8 kind: Event lastTimestamp: "2026-04-24T21:40:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:29Z" name: success-200-isvc-75b0d-predictor-9549449fc-92fhr.18a968ddaf58ba77 namespace: kserve-ci-e2e-test resourceVersion: "20692" uid: 115c461c-4e43-4e79-b2e0-7ddbb06c444b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-75b0d-predictor-9549449fc-92fhr namespace: kserve-ci-e2e-test resourceVersion: "20614" uid: d6580f8e-1f40-4415-b2d8-36099f4708a8 kind: Event lastTimestamp: "2026-04-24T21:40:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:29Z" name: success-200-isvc-75b0d-predictor-9549449fc-92fhr.18a968ddafd1b70f namespace: kserve-ci-e2e-test resourceVersion: "20693" uid: 6a7dc3d0-c85b-4271-9eb9-6a8bb4cb2aff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-75b0d-predictor-9549449fc-92fhr namespace: kserve-ci-e2e-test resourceVersion: "20614" uid: d6580f8e-1f40-4415-b2d8-36099f4708a8 kind: Event lastTimestamp: "2026-04-24T21:40:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:29Z" name: success-200-isvc-75b0d-predictor-9549449fc-92fhr.18a968ddafde44e9 namespace: kserve-ci-e2e-test resourceVersion: "20694" uid: a1603dcc-c6f0-4f4d-b036-20fd7eefdb2a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-75b0d-predictor-9549449fc-92fhr namespace: kserve-ci-e2e-test resourceVersion: "20614" uid: d6580f8e-1f40-4415-b2d8-36099f4708a8 kind: Event lastTimestamp: "2026-04-24T21:40:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:29Z" name: success-200-isvc-75b0d-predictor-9549449fc-92fhr.18a968ddb37bc0c9 namespace: kserve-ci-e2e-test resourceVersion: "20696" uid: 43b34eaf-2efe-4fdb-83b7-a454632d69e5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-75b0d-predictor-9549449fc-92fhr namespace: kserve-ci-e2e-test resourceVersion: "20614" uid: d6580f8e-1f40-4415-b2d8-36099f4708a8 kind: Event lastTimestamp: "2026-04-24T21:40:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:29Z" name: success-200-isvc-75b0d-predictor-9549449fc-92fhr.18a968ddb3f639b4 namespace: kserve-ci-e2e-test resourceVersion: "20697" uid: 1a288fb6-9eca-4f97-9262-48aaff63b97a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:40:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-75b0d-predictor-9549449fc-92fhr namespace: kserve-ci-e2e-test resourceVersion: "20614" uid: d6580f8e-1f40-4415-b2d8-36099f4708a8 kind: Event lastTimestamp: "2026-04-24T21:49:45Z" message: 'Readiness probe failed: dial tcp 10.132.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:40:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:45Z" name: success-200-isvc-75b0d-predictor-9549449fc-92fhr.18a968ddcd1f8313 namespace: kserve-ci-e2e-test resourceVersion: "25056" uid: 05e56f46-6162-431f-8d52-90820e1616bf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-75b0d-predictor-9549449fc-92fhr namespace: kserve-ci-e2e-test resourceVersion: "20614" uid: d6580f8e-1f40-4415-b2d8-36099f4708a8 kind: Event lastTimestamp: "2026-04-24T21:49:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:43Z" name: success-200-isvc-75b0d-predictor-9549449fc-92fhr.18a9695e9a99b21a namespace: kserve-ci-e2e-test resourceVersion: "24938" uid: 7498bba8-b5c6-4a4a-b298-828a8ae50c16 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-75b0d-predictor-9549449fc-92fhr namespace: kserve-ci-e2e-test resourceVersion: "20614" uid: d6580f8e-1f40-4415-b2d8-36099f4708a8 kind: Event lastTimestamp: "2026-04-24T21:49:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:43Z" name: success-200-isvc-75b0d-predictor-9549449fc-92fhr.18a9695e9a9a422b namespace: kserve-ci-e2e-test resourceVersion: "24939" uid: f517c191-c048-462c-acb1-5a79a3b11ad6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:49:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-75b0d-predictor-9549449fc-92fhr namespace: kserve-ci-e2e-test resourceVersion: "20614" uid: d6580f8e-1f40-4415-b2d8-36099f4708a8 kind: Event lastTimestamp: "2026-04-24T21:49:45Z" message: 'Readiness probe failed: Get "https://10.132.0.38:8643/healthz": dial tcp 10.132.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:49:45Z" name: success-200-isvc-75b0d-predictor-9549449fc-92fhr.18a9695f4171ad0b namespace: kserve-ci-e2e-test resourceVersion: "25055" uid: 2c9274b6-0561-425e-9f70-eb15be7c6150 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-75b0d-predictor-9549449fc namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: e7814245-e8c3-4084-a17a-23355ebfc2f2 kind: Event lastTimestamp: "2026-04-24T21:40:28Z" message: 'Created pod: success-200-isvc-75b0d-predictor-9549449fc-92fhr' metadata: creationTimestamp: "2026-04-24T21:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:28Z" name: success-200-isvc-75b0d-predictor-9549449fc.18a968dd6c0174e2 namespace: kserve-ci-e2e-test resourceVersion: "20613" uid: 9605cf80-d106-4c93-a1bb-318583d5f789 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-75b0d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20588" uid: 76bbc05d-e99d-48d1-8516-682808ec8d89 kind: Event lastTimestamp: "2026-04-24T21:40:28Z" message: Scaled up replica set success-200-isvc-75b0d-predictor-9549449fc from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:28Z" name: success-200-isvc-75b0d-predictor.18a968dd68e99f3f namespace: kserve-ci-e2e-test resourceVersion: "20601" uid: 4fe3110d-a142-4d34-98d9-788f3fedf82c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:40:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-75b0d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: 90ff78ca-d540-4df1-8419-e86f36146927 kind: Event lastTimestamp: "2026-04-24T21:41:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:40:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:43Z" name: success-200-isvc-75b0d-predictor.18a968e0e752b845 namespace: kserve-ci-e2e-test resourceVersion: "21556" uid: da9ae41c-c8ec-477a-89e5-b0cd7b7ec1af reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:40:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-75b0d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: 90ff78ca-d540-4df1-8419-e86f36146927 kind: Event lastTimestamp: "2026-04-24T21:41:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:40:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:43Z" name: success-200-isvc-75b0d-predictor.18a968e0e7b31d09 namespace: kserve-ci-e2e-test resourceVersion: "21560" uid: 78afdebc-4ec2-4751-a8fd-8c5f5647ed67 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:41:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-75b0d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20838" uid: 90ff78ca-d540-4df1-8419-e86f36146927 kind: Event lastTimestamp: "2026-04-24T21:41:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:41:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:28Z" name: success-200-isvc-75b0d-predictor.18a968e7e57366b0 namespace: kserve-ci-e2e-test resourceVersion: "21406" uid: 5044da48-41ff-4788-a05e-9fbc46883085 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:41:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-75b0d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20838" uid: 90ff78ca-d540-4df1-8419-e86f36146927 kind: Event lastTimestamp: "2026-04-24T21:41:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:41:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:41:28Z" name: success-200-isvc-75b0d-predictor.18a968e7e5d3efdc namespace: kserve-ci-e2e-test resourceVersion: "21409" uid: a5a4f0f2-76be-443d-9617-7b7d59c21b08 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:40:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-75b0d namespace: kserve-ci-e2e-test resourceVersion: "20580" uid: 593dddc2-43d6-4668-8264-65f9606b0276 kind: Event lastTimestamp: "2026-04-24T21:41:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:41:15Z" name: success-200-isvc-75b0d.18a968dd68bd1a69 namespace: kserve-ci-e2e-test resourceVersion: "21296" uid: 0bb79c2a-e5c7-4f5e-ad0e-771334077a78 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-75b0d namespace: kserve-ci-e2e-test resourceVersion: "20580" uid: 593dddc2-43d6-4668-8264-65f9606b0276 kind: Event lastTimestamp: "2026-04-24T21:40:28Z" message: 'Failed to update status for InferenceService "success-200-isvc-75b0d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-75b0d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:40:28Z" name: success-200-isvc-75b0d.18a968dd698acab8 namespace: kserve-ci-e2e-test resourceVersion: "20605" uid: 4c296f55-a075-4b1a-8c61-1ad6fb6dfb01 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-75b0d namespace: kserve-ci-e2e-test resourceVersion: "20580" uid: 593dddc2-43d6-4668-8264-65f9606b0276 kind: Event lastTimestamp: "2026-04-24T21:40:28Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-75b0d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:40:28Z" name: success-200-isvc-75b0d.18a968dd698b2e38 namespace: kserve-ci-e2e-test resourceVersion: "20608" uid: e25366c8-5fd2-447d-a774-d69bb6bbb453 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-75b0d namespace: kserve-ci-e2e-test resourceVersion: "21292" uid: 593dddc2-43d6-4668-8264-65f9606b0276 kind: Event lastTimestamp: "2026-04-24T21:41:15Z" message: InferenceService [success-200-isvc-75b0d] is Ready metadata: creationTimestamp: "2026-04-24T21:41:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:41:15Z" name: success-200-isvc-75b0d.18a968e8879a2917 namespace: kserve-ci-e2e-test resourceVersion: "21293" uid: 020429a5-cd7d-41c1-b2b5-a8234e5b50eb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:50:19.020154Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-98aac-predictor-7df4fb9989-xwljl namespace: kserve-ci-e2e-test resourceVersion: "25656" uid: 88b92159-1eab-4488-b61c-80aceb69ca9e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-98aac-predictor-7df4fb9989-xwljl to ip-10-0-128-21.ec2.internal metadata: creationTimestamp: "2026-04-24T21:50:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:50:19Z" name: success-200-isvc-98aac-predictor-7df4fb9989-xwljl.18a96966fc9e067e namespace: kserve-ci-e2e-test resourceVersion: "25661" uid: 0f772f08-5eff-458f-b49a-8cfb01663c2b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:19Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-98aac-predictor-7df4fb9989-xwljl namespace: kserve-ci-e2e-test resourceVersion: "25659" uid: 88b92159-1eab-4488-b61c-80aceb69ca9e kind: Event lastTimestamp: "2026-04-24T21:50:19Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-98aac-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:50:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:19Z" name: success-200-isvc-98aac-predictor-7df4fb9989-xwljl.18a969670843bc15 namespace: kserve-ci-e2e-test resourceVersion: "25708" uid: 18583ed5-69c8-4d61-bc1b-8cb845e0c45e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:20Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-98aac-predictor-7df4fb9989-xwljl namespace: kserve-ci-e2e-test resourceVersion: "25670" uid: 88b92159-1eab-4488-b61c-80aceb69ca9e kind: Event lastTimestamp: "2026-04-24T21:50:20Z" message: Add eth0 [10.132.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:50:20Z" name: success-200-isvc-98aac-predictor-7df4fb9989-xwljl.18a9696739c28cbd namespace: kserve-ci-e2e-test resourceVersion: "25722" uid: 703876f2-1126-46fd-8637-6cd821f9c51d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-98aac-predictor-7df4fb9989-xwljl namespace: kserve-ci-e2e-test resourceVersion: "25659" uid: 88b92159-1eab-4488-b61c-80aceb69ca9e kind: Event lastTimestamp: "2026-04-24T21:50:20Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1447" already present on machine metadata: creationTimestamp: "2026-04-24T21:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:20Z" name: success-200-isvc-98aac-predictor-7df4fb9989-xwljl.18a969673b2fc6e6 namespace: kserve-ci-e2e-test resourceVersion: "25725" uid: bc010d02-878c-415f-9da0-81b2d7726f05 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-98aac-predictor-7df4fb9989-xwljl namespace: kserve-ci-e2e-test resourceVersion: "25659" uid: 88b92159-1eab-4488-b61c-80aceb69ca9e kind: Event lastTimestamp: "2026-04-24T21:50:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:20Z" name: success-200-isvc-98aac-predictor-7df4fb9989-xwljl.18a969673ea467c0 namespace: kserve-ci-e2e-test resourceVersion: "25726" uid: 440067bf-b23e-4690-86b6-6f793e49e23a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-98aac-predictor-7df4fb9989-xwljl namespace: kserve-ci-e2e-test resourceVersion: "25659" uid: 88b92159-1eab-4488-b61c-80aceb69ca9e kind: Event lastTimestamp: "2026-04-24T21:50:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:20Z" name: success-200-isvc-98aac-predictor-7df4fb9989-xwljl.18a969673f19eff3 namespace: kserve-ci-e2e-test resourceVersion: "25727" uid: 384c6926-6659-4830-997b-f52af0ae5fc9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-98aac-predictor-7df4fb9989-xwljl namespace: kserve-ci-e2e-test resourceVersion: "25659" uid: 88b92159-1eab-4488-b61c-80aceb69ca9e kind: Event lastTimestamp: "2026-04-24T21:50:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:20Z" name: success-200-isvc-98aac-predictor-7df4fb9989-xwljl.18a969673f27282f namespace: kserve-ci-e2e-test resourceVersion: "25728" uid: 2279f7a0-1cdb-4503-8d6b-903b5346230d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-98aac-predictor-7df4fb9989-xwljl namespace: kserve-ci-e2e-test resourceVersion: "25659" uid: 88b92159-1eab-4488-b61c-80aceb69ca9e kind: Event lastTimestamp: "2026-04-24T21:50:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:20Z" name: success-200-isvc-98aac-predictor-7df4fb9989-xwljl.18a9696742aed8dd namespace: kserve-ci-e2e-test resourceVersion: "25733" uid: 1b9f16e6-4879-4020-8e86-dc78c77e685e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-98aac-predictor-7df4fb9989-xwljl namespace: kserve-ci-e2e-test resourceVersion: "25659" uid: 88b92159-1eab-4488-b61c-80aceb69ca9e kind: Event lastTimestamp: "2026-04-24T21:50:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:20Z" name: success-200-isvc-98aac-predictor-7df4fb9989-xwljl.18a96967432840a8 namespace: kserve-ci-e2e-test resourceVersion: "25734" uid: 9b0f005b-b173-4aba-8947-6a409a133647 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:50:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-98aac-predictor-7df4fb9989-xwljl namespace: kserve-ci-e2e-test resourceVersion: "25659" uid: 88b92159-1eab-4488-b61c-80aceb69ca9e kind: Event lastTimestamp: "2026-04-24T21:50:57Z" message: 'Readiness probe failed: dial tcp 10.132.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:50:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:50:57Z" name: success-200-isvc-98aac-predictor-7df4fb9989-xwljl.18a96967a05e26a2 namespace: kserve-ci-e2e-test resourceVersion: "26261" uid: 5069919c-eb3c-417a-8046-2a78d0a50b5a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-98aac-predictor-7df4fb9989-xwljl namespace: kserve-ci-e2e-test resourceVersion: "25659" uid: 88b92159-1eab-4488-b61c-80aceb69ca9e kind: Event lastTimestamp: "2026-04-24T22:07:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:07:38Z" name: success-200-isvc-98aac-predictor-7df4fb9989-xwljl.18a96a5900f669f0 namespace: kserve-ci-e2e-test resourceVersion: "33394" uid: f051e05b-8c14-45c4-b969-76db5b019b30 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-98aac-predictor-7df4fb9989-xwljl namespace: kserve-ci-e2e-test resourceVersion: "25659" uid: 88b92159-1eab-4488-b61c-80aceb69ca9e kind: Event lastTimestamp: "2026-04-24T22:07:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:07:38Z" name: success-200-isvc-98aac-predictor-7df4fb9989-xwljl.18a96a5900f6e433 namespace: kserve-ci-e2e-test resourceVersion: "33395" uid: 3cbbcb10-d1be-4e4a-9bf5-76804ea4b263 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-98aac-predictor-7df4fb9989 namespace: kserve-ci-e2e-test resourceVersion: "25642" uid: b3bc5c13-4cc9-460a-8f2c-e4c9b7d795f6 kind: Event lastTimestamp: "2026-04-24T21:50:19Z" message: 'Created pod: success-200-isvc-98aac-predictor-7df4fb9989-xwljl' metadata: creationTimestamp: "2026-04-24T21:50:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:19Z" name: success-200-isvc-98aac-predictor-7df4fb9989.18a96966fc27eba6 namespace: kserve-ci-e2e-test resourceVersion: "25657" uid: 80d47f4d-1376-44d4-a874-c92e715c5184 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-98aac-predictor namespace: kserve-ci-e2e-test resourceVersion: "25637" uid: a24c316b-cb5b-4a72-8aef-49c413bbd904 kind: Event lastTimestamp: "2026-04-24T21:50:18Z" message: Scaled up replica set success-200-isvc-98aac-predictor-7df4fb9989 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:50:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:18Z" name: success-200-isvc-98aac-predictor.18a96966f9f03402 namespace: kserve-ci-e2e-test resourceVersion: "25648" uid: f8003d26-4995-4181-af89-e45850617849 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:50:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-98aac-predictor namespace: kserve-ci-e2e-test resourceVersion: "25641" uid: ca9f364a-1fdc-4dfd-9faa-16ac1734efbc kind: Event lastTimestamp: "2026-04-24T21:50:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:50:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:49Z" name: success-200-isvc-98aac-predictor.18a9696a78934577 namespace: kserve-ci-e2e-test resourceVersion: "26042" uid: 12016169-18eb-47d0-95a1-1c3bb1bed42a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:50:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-98aac-predictor namespace: kserve-ci-e2e-test resourceVersion: "25641" uid: ca9f364a-1fdc-4dfd-9faa-16ac1734efbc kind: Event lastTimestamp: "2026-04-24T21:50:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:50:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:50:49Z" name: success-200-isvc-98aac-predictor.18a9696a78f7feff namespace: kserve-ci-e2e-test resourceVersion: "26043" uid: 7da48ebf-4dcb-4a67-a16c-01322670416b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:51:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-98aac-predictor namespace: kserve-ci-e2e-test resourceVersion: "25885" uid: ca9f364a-1fdc-4dfd-9faa-16ac1734efbc kind: Event lastTimestamp: "2026-04-24T21:51:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:51:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:51:34Z" name: success-200-isvc-98aac-predictor.18a9697176c4e241 namespace: kserve-ci-e2e-test resourceVersion: "26611" uid: c1b0375c-2a55-4423-b8c6-9fe6defa1277 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:51:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-98aac-predictor namespace: kserve-ci-e2e-test resourceVersion: "25885" uid: ca9f364a-1fdc-4dfd-9faa-16ac1734efbc kind: Event lastTimestamp: "2026-04-24T21:51:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:51:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:51:34Z" name: success-200-isvc-98aac-predictor.18a96971772ac1b3 namespace: kserve-ci-e2e-test resourceVersion: "26612" uid: 9e50861f-8014-4f3d-828a-1d12a126043e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:50:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-98aac namespace: kserve-ci-e2e-test resourceVersion: "25623" uid: ac549f11-3d2d-4418-b6f7-2ae014505287 kind: Event lastTimestamp: "2026-04-24T21:51:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:50:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:51:07Z" name: success-200-isvc-98aac.18a96966f9f8f2c4 namespace: kserve-ci-e2e-test resourceVersion: "26359" uid: 15908875-4e35-4f11-a742-de9dd681ab30 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-98aac namespace: kserve-ci-e2e-test resourceVersion: "25623" uid: ac549f11-3d2d-4418-b6f7-2ae014505287 kind: Event lastTimestamp: "2026-04-24T21:50:18Z" message: 'Failed to update status for InferenceService "success-200-isvc-98aac": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-98aac": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:50:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:18Z" name: success-200-isvc-98aac.18a96966faa53f25 namespace: kserve-ci-e2e-test resourceVersion: "25651" uid: 31e2a8f5-9eaa-4fc9-be43-b2d7828422a4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:50:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-98aac namespace: kserve-ci-e2e-test resourceVersion: "25623" uid: ac549f11-3d2d-4418-b6f7-2ae014505287 kind: Event lastTimestamp: "2026-04-24T21:50:18Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-98aac": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:50:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:50:18Z" name: success-200-isvc-98aac.18a96966faa598e8 namespace: kserve-ci-e2e-test resourceVersion: "25652" uid: e3be99ed-bc85-4857-a025-ff78d41d9e84 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:51:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-98aac namespace: kserve-ci-e2e-test resourceVersion: "26356" uid: ac549f11-3d2d-4418-b6f7-2ae014505287 kind: Event lastTimestamp: "2026-04-24T21:51:07Z" message: InferenceService [success-200-isvc-98aac] is Ready metadata: creationTimestamp: "2026-04-24T21:51:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:51:07Z" name: success-200-isvc-98aac.18a969725b19fd19 namespace: kserve-ci-e2e-test resourceVersion: "26357" uid: 3178b14a-1a41-45e4-8e39-8252b49b362e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:30:03.194336Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b868d-predictor-6f6bd7f57-hn5g8 namespace: kserve-ci-e2e-test resourceVersion: "15333" uid: 1b444c97-fd1e-4685-8d79-0790643b0ed3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b868d-predictor-6f6bd7f57-hn5g8 to ip-10-0-128-21.ec2.internal metadata: creationTimestamp: "2026-04-24T21:30:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:30:03Z" name: success-200-isvc-b868d-predictor-6f6bd7f57-hn5g8.18a9684be7c03d16 namespace: kserve-ci-e2e-test resourceVersion: "15337" uid: 5459d301-c109-4897-ba31-c6f3b3edc485 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:03Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b868d-predictor-6f6bd7f57-hn5g8 namespace: kserve-ci-e2e-test resourceVersion: "15341" uid: 1b444c97-fd1e-4685-8d79-0790643b0ed3 kind: Event lastTimestamp: "2026-04-24T21:30:03Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:30:03Z" name: success-200-isvc-b868d-predictor-6f6bd7f57-hn5g8.18a9684c013b5a17 namespace: kserve-ci-e2e-test resourceVersion: "15361" uid: b4065d6c-296f-4e32-84be-8d625e780dd8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b868d-predictor-6f6bd7f57-hn5g8 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 1b444c97-fd1e-4685-8d79-0790643b0ed3 kind: Event lastTimestamp: "2026-04-24T21:30:03Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1447" already present on machine metadata: creationTimestamp: "2026-04-24T21:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:03Z" name: success-200-isvc-b868d-predictor-6f6bd7f57-hn5g8.18a9684c0299587b namespace: kserve-ci-e2e-test resourceVersion: "15369" uid: fa1d7224-26c2-4259-8c45-41c9cbeec906 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b868d-predictor-6f6bd7f57-hn5g8 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 1b444c97-fd1e-4685-8d79-0790643b0ed3 kind: Event lastTimestamp: "2026-04-24T21:30:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:03Z" name: success-200-isvc-b868d-predictor-6f6bd7f57-hn5g8.18a9684c056d840c namespace: kserve-ci-e2e-test resourceVersion: "15373" uid: 6c817cde-03d0-442e-a8bb-d0afa474d7ff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b868d-predictor-6f6bd7f57-hn5g8 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 1b444c97-fd1e-4685-8d79-0790643b0ed3 kind: Event lastTimestamp: "2026-04-24T21:30:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:03Z" name: success-200-isvc-b868d-predictor-6f6bd7f57-hn5g8.18a9684c05e0fe36 namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: 565a40ae-745c-4dcd-8a9e-12c2e6bcdb91 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b868d-predictor-6f6bd7f57-hn5g8 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 1b444c97-fd1e-4685-8d79-0790643b0ed3 kind: Event lastTimestamp: "2026-04-24T21:30:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:03Z" name: success-200-isvc-b868d-predictor-6f6bd7f57-hn5g8.18a9684c05ed5cde namespace: kserve-ci-e2e-test resourceVersion: "15375" uid: c50f8302-fde2-4801-9482-22cdbbfa9f82 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b868d-predictor-6f6bd7f57-hn5g8 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 1b444c97-fd1e-4685-8d79-0790643b0ed3 kind: Event lastTimestamp: "2026-04-24T21:30:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:03Z" name: success-200-isvc-b868d-predictor-6f6bd7f57-hn5g8.18a9684c08b11fd2 namespace: kserve-ci-e2e-test resourceVersion: "15377" uid: 94bb8504-ef14-4ba5-8ce9-05cf85a5cda0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b868d-predictor-6f6bd7f57-hn5g8 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 1b444c97-fd1e-4685-8d79-0790643b0ed3 kind: Event lastTimestamp: "2026-04-24T21:30:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:03Z" name: success-200-isvc-b868d-predictor-6f6bd7f57-hn5g8.18a9684c0922e95d namespace: kserve-ci-e2e-test resourceVersion: "15379" uid: e16c6aff-357e-47bd-95a7-ceb0caa5f41a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T21:30:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b868d-predictor-6f6bd7f57-hn5g8 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 1b444c97-fd1e-4685-8d79-0790643b0ed3 kind: Event lastTimestamp: "2026-04-24T21:39:20Z" message: 'Readiness probe failed: dial tcp 10.132.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:30:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:20Z" name: success-200-isvc-b868d-predictor-6f6bd7f57-hn5g8.18a9684c4487971f namespace: kserve-ci-e2e-test resourceVersion: "19810" uid: dff7bf94-12f2-43e0-b96b-92777a899b03 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b868d-predictor-6f6bd7f57-hn5g8 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 1b444c97-fd1e-4685-8d79-0790643b0ed3 kind: Event lastTimestamp: "2026-04-24T21:39:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:17Z" name: success-200-isvc-b868d-predictor-6f6bd7f57-hn5g8.18a968cd0d8d7150 namespace: kserve-ci-e2e-test resourceVersion: "19697" uid: 0bd7169b-7d34-41ed-84f4-a2a9c03e9713 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b868d-predictor-6f6bd7f57-hn5g8 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 1b444c97-fd1e-4685-8d79-0790643b0ed3 kind: Event lastTimestamp: "2026-04-24T21:39:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:17Z" name: success-200-isvc-b868d-predictor-6f6bd7f57-hn5g8.18a968cd0d8e19c8 namespace: kserve-ci-e2e-test resourceVersion: "19700" uid: 9818696e-1cd8-46e7-b1d0-fd8f952e2468 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b868d-predictor-6f6bd7f57-hn5g8 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 1b444c97-fd1e-4685-8d79-0790643b0ed3 kind: Event lastTimestamp: "2026-04-24T21:39:20Z" message: 'Readiness probe failed: Get "https://10.132.0.30:8643/healthz": dial tcp 10.132.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:39:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:20Z" name: success-200-isvc-b868d-predictor-6f6bd7f57-hn5g8.18a968cdb8dd7eff namespace: kserve-ci-e2e-test resourceVersion: "19808" uid: ccf80acf-bbba-4267-8d56-beac7cb75e7c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b868d-predictor-6f6bd7f57 namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: bd8f7fc0-1a7f-4efa-a616-ab699324bdb5 kind: Event lastTimestamp: "2026-04-24T21:30:03Z" message: 'Created pod: success-200-isvc-b868d-predictor-6f6bd7f57-hn5g8' metadata: creationTimestamp: "2026-04-24T21:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:03Z" name: success-200-isvc-b868d-predictor-6f6bd7f57.18a9684be72c3a1b namespace: kserve-ci-e2e-test resourceVersion: "15334" uid: 73efc0af-aa7e-4dcb-b365-3d11e63379d7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b868d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15299" uid: 2af6eb05-e041-4b90-8e04-8f370e82a61c kind: Event lastTimestamp: "2026-04-24T21:30:02Z" message: Scaled up replica set success-200-isvc-b868d-predictor-6f6bd7f57 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:03Z" name: success-200-isvc-b868d-predictor.18a9684bdc2055d6 namespace: kserve-ci-e2e-test resourceVersion: "15305" uid: 09963872-a424-4906-bd7c-89b11b7a2980 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:30:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b868d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15301" uid: 258d8177-b0ac-4dc0-a334-51b723017d76 kind: Event lastTimestamp: "2026-04-24T21:30:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:48Z" name: success-200-isvc-b868d-predictor.18a9684f5aa5a5f5 namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 0614c761-e328-49de-8ed6-a93643616917 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:30:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b868d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15301" uid: 258d8177-b0ac-4dc0-a334-51b723017d76 kind: Event lastTimestamp: "2026-04-24T21:30:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:30:48Z" name: success-200-isvc-b868d-predictor.18a9684f5aff6659 namespace: kserve-ci-e2e-test resourceVersion: "15999" uid: 1a5ca836-35b0-482a-88ed-48486dbc8838 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:31:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b868d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15534" uid: 258d8177-b0ac-4dc0-a334-51b723017d76 kind: Event lastTimestamp: "2026-04-24T21:31:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:31:18Z" name: success-200-isvc-b868d-predictor.18a96859d7a698b9 namespace: kserve-ci-e2e-test resourceVersion: "16294" uid: 3abc5737-19c3-4900-aade-b7e97df459f7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:31:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b868d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15534" uid: 258d8177-b0ac-4dc0-a334-51b723017d76 kind: Event lastTimestamp: "2026-04-24T21:31:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:31:18Z" name: success-200-isvc-b868d-predictor.18a96859d8093de7 namespace: kserve-ci-e2e-test resourceVersion: "16295" uid: 901da2f4-1ab6-4ddb-95b8-bf7892541744 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:30:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b868d namespace: kserve-ci-e2e-test resourceVersion: "15280" uid: 1360e4d1-ac0d-4fde-8041-255fea901f2f kind: Event lastTimestamp: "2026-04-24T21:30:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:30:50Z" name: success-200-isvc-b868d.18a9684bdc14338a namespace: kserve-ci-e2e-test resourceVersion: "16028" uid: 4a623d76-259e-45f1-857c-0d93bb1cf92e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b868d namespace: kserve-ci-e2e-test resourceVersion: "15280" uid: 1360e4d1-ac0d-4fde-8041-255fea901f2f kind: Event lastTimestamp: "2026-04-24T21:30:03Z" message: 'Failed to update status for InferenceService "success-200-isvc-b868d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b868d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:30:03Z" name: success-200-isvc-b868d.18a9684bdcc214b1 namespace: kserve-ci-e2e-test resourceVersion: "15309" uid: 0859cf4e-b1ce-43bc-8971-853eaa3c3f55 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b868d namespace: kserve-ci-e2e-test resourceVersion: "15280" uid: 1360e4d1-ac0d-4fde-8041-255fea901f2f kind: Event lastTimestamp: "2026-04-24T21:30:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b868d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:30:03Z" name: success-200-isvc-b868d.18a9684bdcc254f6 namespace: kserve-ci-e2e-test resourceVersion: "15330" uid: 8f47fc1a-6ff9-458d-b4e4-ab0d91554d56 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b868d namespace: kserve-ci-e2e-test resourceVersion: "16024" uid: 1360e4d1-ac0d-4fde-8041-255fea901f2f kind: Event lastTimestamp: "2026-04-24T21:30:50Z" message: InferenceService [success-200-isvc-b868d] is Ready metadata: creationTimestamp: "2026-04-24T21:30:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:30:50Z" name: success-200-isvc-b868d.18a96856ff67fcd4 namespace: kserve-ci-e2e-test resourceVersion: "16025" uid: 093890fb-2387-45ba-8829-b7cd4a9a2f5e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:39:53.932113Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c1a35-predictor-869c6689b7-jfwbk namespace: kserve-ci-e2e-test resourceVersion: "20133" uid: c2b3f2bd-2603-4bff-8cdc-d7717704beab kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c1a35-predictor-869c6689b7-jfwbk to ip-10-0-128-21.ec2.internal metadata: creationTimestamp: "2026-04-24T21:39:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:39:53Z" name: success-200-isvc-c1a35-predictor-869c6689b7-jfwbk.18a968d572776dc8 namespace: kserve-ci-e2e-test resourceVersion: "20138" uid: 39c2a3e6-1b2c-4dda-9900-d39182b3ac9f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:54Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c1a35-predictor-869c6689b7-jfwbk namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: c2b3f2bd-2603-4bff-8cdc-d7717704beab kind: Event lastTimestamp: "2026-04-24T21:39:54Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-c1a35-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:39:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:54Z" name: success-200-isvc-c1a35-predictor-869c6689b7-jfwbk.18a968d5844f9bf9 namespace: kserve-ci-e2e-test resourceVersion: "20185" uid: 2bd7dc02-3ad0-4865-be5a-331f1b6b22d9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:54Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c1a35-predictor-869c6689b7-jfwbk namespace: kserve-ci-e2e-test resourceVersion: "20152" uid: c2b3f2bd-2603-4bff-8cdc-d7717704beab kind: Event lastTimestamp: "2026-04-24T21:39:54Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:39:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:39:54Z" name: success-200-isvc-c1a35-predictor-869c6689b7-jfwbk.18a968d5b0224c80 namespace: kserve-ci-e2e-test resourceVersion: "20197" uid: 39d5ee80-2520-40c2-b973-54e87d5b8d04 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c1a35-predictor-869c6689b7-jfwbk namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: c2b3f2bd-2603-4bff-8cdc-d7717704beab kind: Event lastTimestamp: "2026-04-24T21:39:54Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1447" already present on machine metadata: creationTimestamp: "2026-04-24T21:39:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:54Z" name: success-200-isvc-c1a35-predictor-869c6689b7-jfwbk.18a968d5b1cd6197 namespace: kserve-ci-e2e-test resourceVersion: "20199" uid: ad63f7f5-b862-49df-a8e7-d7e7e5d5ad6a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c1a35-predictor-869c6689b7-jfwbk namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: c2b3f2bd-2603-4bff-8cdc-d7717704beab kind: Event lastTimestamp: "2026-04-24T21:39:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:55Z" name: success-200-isvc-c1a35-predictor-869c6689b7-jfwbk.18a968d5b4e5e830 namespace: kserve-ci-e2e-test resourceVersion: "20201" uid: 1c8f1f1f-b0b8-4660-99ce-da2c26e87f6b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c1a35-predictor-869c6689b7-jfwbk namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: c2b3f2bd-2603-4bff-8cdc-d7717704beab kind: Event lastTimestamp: "2026-04-24T21:39:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:55Z" name: success-200-isvc-c1a35-predictor-869c6689b7-jfwbk.18a968d5b55f2532 namespace: kserve-ci-e2e-test resourceVersion: "20203" uid: 898afa9c-fa99-4a37-9982-6f49cc7b44fc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c1a35-predictor-869c6689b7-jfwbk namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: c2b3f2bd-2603-4bff-8cdc-d7717704beab kind: Event lastTimestamp: "2026-04-24T21:39:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:55Z" name: success-200-isvc-c1a35-predictor-869c6689b7-jfwbk.18a968d5b56b351b namespace: kserve-ci-e2e-test resourceVersion: "20205" uid: e36b8843-1359-4d1a-bdc9-88d57abde2e2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c1a35-predictor-869c6689b7-jfwbk namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: c2b3f2bd-2603-4bff-8cdc-d7717704beab kind: Event lastTimestamp: "2026-04-24T21:39:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:55Z" name: success-200-isvc-c1a35-predictor-869c6689b7-jfwbk.18a968d5b8768814 namespace: kserve-ci-e2e-test resourceVersion: "20207" uid: 5ea7fd23-63ff-4dbd-9d26-62f9cdb498bd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c1a35-predictor-869c6689b7-jfwbk namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: c2b3f2bd-2603-4bff-8cdc-d7717704beab kind: Event lastTimestamp: "2026-04-24T21:39:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:55Z" name: success-200-isvc-c1a35-predictor-869c6689b7-jfwbk.18a968d5b9309c90 namespace: kserve-ci-e2e-test resourceVersion: "20208" uid: e071ad1d-7ffd-45ed-bb4e-c23c0e599f10 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:39:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c1a35-predictor-869c6689b7-jfwbk namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: c2b3f2bd-2603-4bff-8cdc-d7717704beab kind: Event lastTimestamp: "2026-04-24T21:40:32Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:39:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:32Z" name: success-200-isvc-c1a35-predictor-869c6689b7-jfwbk.18a968d617699bcc namespace: kserve-ci-e2e-test resourceVersion: "20739" uid: 69466369-5627-46fe-8af7-72624c4f8c25 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c1a35-predictor-869c6689b7-jfwbk namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: c2b3f2bd-2603-4bff-8cdc-d7717704beab kind: Event lastTimestamp: "2026-04-24T21:41:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:04Z" name: success-200-isvc-c1a35-predictor-869c6689b7-jfwbk.18a968e5c95614f8 namespace: kserve-ci-e2e-test resourceVersion: "21111" uid: 7be6e59e-360a-442a-943d-605de4f020bb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:41:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c1a35-predictor-869c6689b7-jfwbk namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: c2b3f2bd-2603-4bff-8cdc-d7717704beab kind: Event lastTimestamp: "2026-04-24T21:41:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:41:04Z" name: success-200-isvc-c1a35-predictor-869c6689b7-jfwbk.18a968e5c95694b6 namespace: kserve-ci-e2e-test resourceVersion: "21113" uid: 54a00ed1-2679-40bd-afb2-9d89593729bf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c1a35-predictor-869c6689b7 namespace: kserve-ci-e2e-test resourceVersion: "20120" uid: 53f2d1cf-0e85-4565-abd3-9408c027e734 kind: Event lastTimestamp: "2026-04-24T21:39:53Z" message: 'Created pod: success-200-isvc-c1a35-predictor-869c6689b7-jfwbk' metadata: creationTimestamp: "2026-04-24T21:39:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:53Z" name: success-200-isvc-c1a35-predictor-869c6689b7.18a968d571fd52d3 namespace: kserve-ci-e2e-test resourceVersion: "20135" uid: d956a818-f601-4d02-8e75-b0810b26ef60 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c1a35-predictor namespace: kserve-ci-e2e-test resourceVersion: "20115" uid: 2089515b-dce4-48ee-9002-6e4c98974e2f kind: Event lastTimestamp: "2026-04-24T21:39:53Z" message: Scaled up replica set success-200-isvc-c1a35-predictor-869c6689b7 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:39:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:53Z" name: success-200-isvc-c1a35-predictor.18a968d56f392986 namespace: kserve-ci-e2e-test resourceVersion: "20125" uid: 27497134-3bb8-45c6-be36-905c37825cac reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c1a35-predictor namespace: kserve-ci-e2e-test resourceVersion: "20119" uid: eb619673-7a96-4261-b97a-fd6f4f7f0dca kind: Event lastTimestamp: "2026-04-24T21:40:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:40:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:08Z" name: success-200-isvc-c1a35-predictor.18a968d8edbeb8cf namespace: kserve-ci-e2e-test resourceVersion: "20361" uid: 37149dde-2487-4267-a17a-cf8c44f1167e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c1a35-predictor namespace: kserve-ci-e2e-test resourceVersion: "20119" uid: eb619673-7a96-4261-b97a-fd6f4f7f0dca kind: Event lastTimestamp: "2026-04-24T21:40:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:40:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:08Z" name: success-200-isvc-c1a35-predictor.18a968d8ee32527b namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: fe64bf08-cc6a-49fa-b7ad-12aaf342eda9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:40:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c1a35-predictor namespace: kserve-ci-e2e-test resourceVersion: "20362" uid: eb619673-7a96-4261-b97a-fd6f4f7f0dca kind: Event lastTimestamp: "2026-04-24T21:40:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:40:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:53Z" name: success-200-isvc-c1a35-predictor.18a968dc6debe81c namespace: kserve-ci-e2e-test resourceVersion: "20942" uid: a466a3f9-420c-4cc6-b005-f8aa05910e1d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:40:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c1a35-predictor namespace: kserve-ci-e2e-test resourceVersion: "20362" uid: eb619673-7a96-4261-b97a-fd6f4f7f0dca kind: Event lastTimestamp: "2026-04-24T21:40:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:40:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:53Z" name: success-200-isvc-c1a35-predictor.18a968dc6ec0cdce namespace: kserve-ci-e2e-test resourceVersion: "20944" uid: 71c4ee6c-2820-4909-a55c-d24666491ea7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:39:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c1a35 namespace: kserve-ci-e2e-test resourceVersion: "20100" uid: 96f5844c-9c6f-4b6a-b0d4-fe6ae1da154e kind: Event lastTimestamp: "2026-04-24T21:40:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:39:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:40:42Z" name: success-200-isvc-c1a35.18a968d56f099645 namespace: kserve-ci-e2e-test resourceVersion: "20832" uid: a7ec8d31-022d-467f-9bd8-c961faa4a9db reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c1a35 namespace: kserve-ci-e2e-test resourceVersion: "20100" uid: 96f5844c-9c6f-4b6a-b0d4-fe6ae1da154e kind: Event lastTimestamp: "2026-04-24T21:39:53Z" message: 'Failed to update status for InferenceService "success-200-isvc-c1a35": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c1a35": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:39:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:39:53Z" name: success-200-isvc-c1a35.18a968d56feec29b namespace: kserve-ci-e2e-test resourceVersion: "20128" uid: f729f512-f739-4960-8976-d7930e3caf86 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c1a35 namespace: kserve-ci-e2e-test resourceVersion: "20100" uid: 96f5844c-9c6f-4b6a-b0d4-fe6ae1da154e kind: Event lastTimestamp: "2026-04-24T21:39:53Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c1a35": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:39:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:39:53Z" name: success-200-isvc-c1a35.18a968d56fef1fc9 namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 70a20b53-f0d2-4ecd-98f4-3a69704a0469 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c1a35 namespace: kserve-ci-e2e-test resourceVersion: "20829" uid: 96f5844c-9c6f-4b6a-b0d4-fe6ae1da154e kind: Event lastTimestamp: "2026-04-24T21:40:42Z" message: InferenceService [success-200-isvc-c1a35] is Ready metadata: creationTimestamp: "2026-04-24T21:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:40:42Z" name: success-200-isvc-c1a35.18a968e0d1fcb53b namespace: kserve-ci-e2e-test resourceVersion: "20830" uid: 4ac97b8f-d9ae-4db4-9808-d73d277e45e4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:28:28.737626Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2 namespace: kserve-ci-e2e-test resourceVersion: "14180" uid: 7b76ea09-17c8-405a-affa-0054342b9b15 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2 to ip-10-0-128-21.ec2.internal metadata: creationTimestamp: "2026-04-24T21:28:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:28:28Z" name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2.18a96835e9b136cb namespace: kserve-ci-e2e-test resourceVersion: "14186" uid: 93c4be7f-423c-4752-9105-00eeb4f19a47 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:29Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2 namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: 7b76ea09-17c8-405a-affa-0054342b9b15 kind: Event lastTimestamp: "2026-04-24T21:28:29Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:28:29Z" name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2.18a9683603704773 namespace: kserve-ci-e2e-test resourceVersion: "14267" uid: 649214d0-9a89-4bf0-8b9c-6f7863c7d866 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2 namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7b76ea09-17c8-405a-affa-0054342b9b15 kind: Event lastTimestamp: "2026-04-24T21:28:29Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1447" metadata: creationTimestamp: "2026-04-24T21:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:29Z" name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2.18a9683604d472dd namespace: kserve-ci-e2e-test resourceVersion: "14270" uid: c7969f31-6b15-4e20-85e9-1fc12cea731e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2 namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7b76ea09-17c8-405a-affa-0054342b9b15 kind: Event lastTimestamp: "2026-04-24T21:28:42Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1447" in 12.921s (12.921s including waiting). Image size: 1335723117 bytes.' metadata: creationTimestamp: "2026-04-24T21:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:42Z" name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2.18a9683906fd4449 namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: 8b8bb0fc-ba6d-4033-b136-875cb14b45a4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2 namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7b76ea09-17c8-405a-affa-0054342b9b15 kind: Event lastTimestamp: "2026-04-24T21:28:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:42Z" name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2.18a968390a47b323 namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: 350c23cc-374c-41a7-868e-365e2ed9090b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2 namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7b76ea09-17c8-405a-affa-0054342b9b15 kind: Event lastTimestamp: "2026-04-24T21:28:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:42Z" name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2.18a968390ac9e453 namespace: kserve-ci-e2e-test resourceVersion: "14441" uid: f9f37286-9106-4aff-a2b5-cb08608f03b6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2 namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7b76ea09-17c8-405a-affa-0054342b9b15 kind: Event lastTimestamp: "2026-04-24T21:28:42Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T21:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:42Z" name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2.18a968390ad6bbcd namespace: kserve-ci-e2e-test resourceVersion: "14442" uid: eaa3f811-95d0-4ba6-be31-1c112c9e4dd7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2 namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7b76ea09-17c8-405a-affa-0054342b9b15 kind: Event lastTimestamp: "2026-04-24T21:28:44Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.487s (2.487s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T21:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:44Z" name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2.18a968399f1f59cb namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: b1dbdf3b-ece9-4e79-858f-b33ff97c11db reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2 namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7b76ea09-17c8-405a-affa-0054342b9b15 kind: Event lastTimestamp: "2026-04-24T21:28:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:44Z" name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2.18a96839a281c740 namespace: kserve-ci-e2e-test resourceVersion: "14490" uid: 2eedc425-bbba-4b0e-9c17-a66fcb3a4a79 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2 namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7b76ea09-17c8-405a-affa-0054342b9b15 kind: Event lastTimestamp: "2026-04-24T21:28:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:28:44Z" name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2.18a96839a2fcbaa2 namespace: kserve-ci-e2e-test resourceVersion: "14491" uid: 50d1c494-04b5-4e19-80fe-f0f37ffa6828 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:28:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2 namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7b76ea09-17c8-405a-affa-0054342b9b15 kind: Event lastTimestamp: "2026-04-24T21:29:21Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:28:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:29:21Z" name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2.18a96839d0c71228 namespace: kserve-ci-e2e-test resourceVersion: "14847" uid: fcb4f86f-b257-4974-8806-1b5bb0c55e1f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2 namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7b76ea09-17c8-405a-affa-0054342b9b15 kind: Event lastTimestamp: "2026-04-24T21:30:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:03Z" name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2.18a9684bdc229124 namespace: kserve-ci-e2e-test resourceVersion: "15306" uid: b0ebd5a8-1183-444e-9e6b-ebe7ba6fd78c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2 namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7b76ea09-17c8-405a-affa-0054342b9b15 kind: Event lastTimestamp: "2026-04-24T21:30:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:03Z" name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2.18a9684bdc234e4d namespace: kserve-ci-e2e-test resourceVersion: "15308" uid: 35c31886-2072-498a-bb25-6a7d3d8d100b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2 namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 7b76ea09-17c8-405a-affa-0054342b9b15 kind: Event lastTimestamp: "2026-04-24T21:30:07Z" message: 'Readiness probe failed: Get "https://10.132.0.27:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-04-24T21:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:07Z" name: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2.18a9684ce890f917 namespace: kserve-ci-e2e-test resourceVersion: "15443" uid: 78a26b6d-4bfe-46e4-9ad3-2525acfd06f0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d41b1-predictor-6fd5c66657 namespace: kserve-ci-e2e-test resourceVersion: "14166" uid: d2ebf598-9461-4cbd-b0ed-26dbf59ee183 kind: Event lastTimestamp: "2026-04-24T21:28:28Z" message: 'Created pod: success-200-isvc-d41b1-predictor-6fd5c66657-8qwq2' metadata: creationTimestamp: "2026-04-24T21:28:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:28:28Z" name: success-200-isvc-d41b1-predictor-6fd5c66657.18a96835e943ddce namespace: kserve-ci-e2e-test resourceVersion: "14181" uid: 1929e694-c642-46d5-ad3d-f5c49c3578f1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d41b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14164" uid: 75832194-df98-462b-89da-baa8fef96c2c kind: Event lastTimestamp: "2026-04-24T21:28:28Z" message: Scaled up replica set success-200-isvc-d41b1-predictor-6fd5c66657 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:28:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:28:28Z" name: success-200-isvc-d41b1-predictor.18a96835e2394f54 namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: 0ce86fbf-0773-40b0-abc1-c640ec39a9b9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:28:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d41b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14177" uid: 7848293a-7aaf-4c30-8021-c520bbef8408 kind: Event lastTimestamp: "2026-04-24T21:28:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:28:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:28:58Z" name: success-200-isvc-d41b1-predictor.18a9683967f48351 namespace: kserve-ci-e2e-test resourceVersion: "14651" uid: 072faf24-318d-44bc-a244-1edb6d09765b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:28:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d41b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14177" uid: 7848293a-7aaf-4c30-8021-c520bbef8408 kind: Event lastTimestamp: "2026-04-24T21:28:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:28:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:28:58Z" name: success-200-isvc-d41b1-predictor.18a9683968586e54 namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 727a15f1-56e0-4a10-9666-41fd8088aaa7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:29:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d41b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14462" uid: 7848293a-7aaf-4c30-8021-c520bbef8408 kind: Event lastTimestamp: "2026-04-24T21:29:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:29:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:29:58Z" name: success-200-isvc-d41b1-predictor.18a96840663f3476 namespace: kserve-ci-e2e-test resourceVersion: "15199" uid: 1c97d159-c0a4-4621-a522-129cdbe06eac reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T21:29:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d41b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14462" uid: 7848293a-7aaf-4c30-8021-c520bbef8408 kind: Event lastTimestamp: "2026-04-24T21:29:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:29:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:29:58Z" name: success-200-isvc-d41b1-predictor.18a9684066a2e8b4 namespace: kserve-ci-e2e-test resourceVersion: "15200" uid: 60cf46f2-0a4f-4e6e-acf3-1871e477fbbd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:28:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d41b1 namespace: kserve-ci-e2e-test resourceVersion: "14155" uid: 777401dd-01b5-474c-97a5-19663aa95aa7 kind: Event lastTimestamp: "2026-04-24T21:29:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:28:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:29:31Z" name: success-200-isvc-d41b1.18a96835e868f03d namespace: kserve-ci-e2e-test resourceVersion: "14943" uid: 4123d769-f19e-4adb-af01-a5403543b404 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d41b1 namespace: kserve-ci-e2e-test resourceVersion: "14155" uid: 777401dd-01b5-474c-97a5-19663aa95aa7 kind: Event lastTimestamp: "2026-04-24T21:28:28Z" message: 'Failed to update status for InferenceService "success-200-isvc-d41b1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d41b1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:28:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:28:28Z" name: success-200-isvc-d41b1.18a96835e967c30b namespace: kserve-ci-e2e-test resourceVersion: "14185" uid: 3ae1d541-0dae-4420-bb1e-2cef36778b68 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:28:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d41b1 namespace: kserve-ci-e2e-test resourceVersion: "14155" uid: 777401dd-01b5-474c-97a5-19663aa95aa7 kind: Event lastTimestamp: "2026-04-24T21:28:28Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d41b1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:28:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:28:28Z" name: success-200-isvc-d41b1.18a96835e9680916 namespace: kserve-ci-e2e-test resourceVersion: "14188" uid: 27d029d7-d4e5-4f10-8395-edac1c313716 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d41b1 namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: 777401dd-01b5-474c-97a5-19663aa95aa7 kind: Event lastTimestamp: "2026-04-24T21:29:31Z" message: InferenceService [success-200-isvc-d41b1] is Ready metadata: creationTimestamp: "2026-04-24T21:29:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:29:31Z" name: success-200-isvc-d41b1.18a968448b5e8c5c namespace: kserve-ci-e2e-test resourceVersion: "14942" uid: f53626a6-31c8-41ae-8ced-199dcac27cce reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:39:17.968216Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f3231-predictor-788854545f-lsqvw namespace: kserve-ci-e2e-test resourceVersion: "19719" uid: 13999f24-2195-46ee-a8de-d2687ebef412 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f3231-predictor-788854545f-lsqvw to ip-10-0-128-21.ec2.internal metadata: creationTimestamp: "2026-04-24T21:39:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:39:17Z" name: success-200-isvc-f3231-predictor-788854545f-lsqvw.18a968cd12d9d6a4 namespace: kserve-ci-e2e-test resourceVersion: "19729" uid: 9721c8b8-34d9-4d19-b4d6-f088940be9a3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:18Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f3231-predictor-788854545f-lsqvw namespace: kserve-ci-e2e-test resourceVersion: "19741" uid: 13999f24-2195-46ee-a8de-d2687ebef412 kind: Event lastTimestamp: "2026-04-24T21:39:18Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:39:18Z" name: success-200-isvc-f3231-predictor-788854545f-lsqvw.18a968cd2cdb30d9 namespace: kserve-ci-e2e-test resourceVersion: "19766" uid: 2ce9348e-f218-4754-b1b6-7cc4217e6675 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f3231-predictor-788854545f-lsqvw namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: 13999f24-2195-46ee-a8de-d2687ebef412 kind: Event lastTimestamp: "2026-04-24T21:39:18Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1447" already present on machine metadata: creationTimestamp: "2026-04-24T21:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:18Z" name: success-200-isvc-f3231-predictor-788854545f-lsqvw.18a968cd2e32c746 namespace: kserve-ci-e2e-test resourceVersion: "19768" uid: 92c48f3e-0237-4ea6-976d-35a79937c8c5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f3231-predictor-788854545f-lsqvw namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: 13999f24-2195-46ee-a8de-d2687ebef412 kind: Event lastTimestamp: "2026-04-24T21:39:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T21:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:18Z" name: success-200-isvc-f3231-predictor-788854545f-lsqvw.18a968cd3123a56b namespace: kserve-ci-e2e-test resourceVersion: "19770" uid: b2c0104c-5e4c-4f11-867c-a0377fc68c38 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f3231-predictor-788854545f-lsqvw namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: 13999f24-2195-46ee-a8de-d2687ebef412 kind: Event lastTimestamp: "2026-04-24T21:39:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T21:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:18Z" name: success-200-isvc-f3231-predictor-788854545f-lsqvw.18a968cd319dd8e0 namespace: kserve-ci-e2e-test resourceVersion: "19771" uid: cbd7e3b8-2633-4438-8ef5-8a8ee94cec5e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f3231-predictor-788854545f-lsqvw namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: 13999f24-2195-46ee-a8de-d2687ebef412 kind: Event lastTimestamp: "2026-04-24T21:39:18Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T21:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:18Z" name: success-200-isvc-f3231-predictor-788854545f-lsqvw.18a968cd31aae470 namespace: kserve-ci-e2e-test resourceVersion: "19772" uid: 03e196b7-c53c-48b5-9f85-e4c6ef327a94 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f3231-predictor-788854545f-lsqvw namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: 13999f24-2195-46ee-a8de-d2687ebef412 kind: Event lastTimestamp: "2026-04-24T21:39:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T21:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:18Z" name: success-200-isvc-f3231-predictor-788854545f-lsqvw.18a968cd357f2a6b namespace: kserve-ci-e2e-test resourceVersion: "19774" uid: 527621a1-9972-4a59-82b0-43baba1e3909 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f3231-predictor-788854545f-lsqvw namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: 13999f24-2195-46ee-a8de-d2687ebef412 kind: Event lastTimestamp: "2026-04-24T21:39:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:18Z" name: success-200-isvc-f3231-predictor-788854545f-lsqvw.18a968cd35fd0313 namespace: kserve-ci-e2e-test resourceVersion: "19775" uid: b472fe5f-0525-41d7-a02f-acb982a39f52 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f3231-predictor-788854545f-lsqvw namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: 13999f24-2195-46ee-a8de-d2687ebef412 kind: Event lastTimestamp: "2026-04-24T21:39:55Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:55Z" name: success-200-isvc-f3231-predictor-788854545f-lsqvw.18a968cd720fbedf namespace: kserve-ci-e2e-test resourceVersion: "20221" uid: 9e8ef14a-bb20-40a9-8a46-adc6fe806b89 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f3231-predictor-788854545f-lsqvw namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: 13999f24-2195-46ee-a8de-d2687ebef412 kind: Event lastTimestamp: "2026-04-24T21:40:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T21:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:28Z" name: success-200-isvc-f3231-predictor-788854545f-lsqvw.18a968dd6a316106 namespace: kserve-ci-e2e-test resourceVersion: "20609" uid: 05300a62-6431-41c5-a2fc-694b63a8bfec reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f3231-predictor-788854545f-lsqvw namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: 13999f24-2195-46ee-a8de-d2687ebef412 kind: Event lastTimestamp: "2026-04-24T21:40:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T21:40:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:28Z" name: success-200-isvc-f3231-predictor-788854545f-lsqvw.18a968dd6a3253e5 namespace: kserve-ci-e2e-test resourceVersion: "20610" uid: 3997d828-5425-4e12-bf0d-edcc2a277e5d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f3231-predictor-788854545f-lsqvw namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: 13999f24-2195-46ee-a8de-d2687ebef412 kind: Event lastTimestamp: "2026-04-24T21:40:30Z" message: 'Readiness probe failed: Get "https://10.132.0.35:8643/healthz": dial tcp 10.132.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T21:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:40:30Z" name: success-200-isvc-f3231-predictor-788854545f-lsqvw.18a968ddfa1bebe8 namespace: kserve-ci-e2e-test resourceVersion: "20714" uid: 90c20203-e168-40f1-a9a4-4526fe5c825a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f3231-predictor-788854545f namespace: kserve-ci-e2e-test resourceVersion: "19698" uid: 61bb554f-84c1-48a1-b237-7f77fa6d9b81 kind: Event lastTimestamp: "2026-04-24T21:39:17Z" message: 'Created pod: success-200-isvc-f3231-predictor-788854545f-lsqvw' metadata: creationTimestamp: "2026-04-24T21:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:17Z" name: success-200-isvc-f3231-predictor-788854545f.18a968cd1234d781 namespace: kserve-ci-e2e-test resourceVersion: "19721" uid: c5db5d13-2d15-4bd3-82c0-a6b0fa947d3e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f3231-predictor namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 5f6f7b10-aef6-43f9-9243-8a2193fd1790 kind: Event lastTimestamp: "2026-04-24T21:39:17Z" message: Scaled up replica set success-200-isvc-f3231-predictor-788854545f from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:39:17Z" name: success-200-isvc-f3231-predictor.18a968cd0e11a942 namespace: kserve-ci-e2e-test resourceVersion: "19701" uid: 7eef65af-c918-4068-b745-2e78b3de2131 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:39:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f3231-predictor namespace: kserve-ci-e2e-test resourceVersion: "19696" uid: fcd80599-4396-4769-8fb5-1eb15173a371 kind: Event lastTimestamp: "2026-04-24T21:40:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:39:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:03Z" name: success-200-isvc-f3231-predictor.18a968d08d4578cd namespace: kserve-ci-e2e-test resourceVersion: "20298" uid: a9d831d3-1541-4b60-8c4a-f360222502ab reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:39:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f3231-predictor namespace: kserve-ci-e2e-test resourceVersion: "19696" uid: fcd80599-4396-4769-8fb5-1eb15173a371 kind: Event lastTimestamp: "2026-04-24T21:40:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:39:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:03Z" name: success-200-isvc-f3231-predictor.18a968d08de9ff54 namespace: kserve-ci-e2e-test resourceVersion: "20299" uid: c48ec8dc-cfb7-4610-91bd-ee71ac0608b7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:39:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f3231-predictor namespace: kserve-ci-e2e-test resourceVersion: "19916" uid: fcd80599-4396-4769-8fb5-1eb15173a371 kind: Event lastTimestamp: "2026-04-24T21:40:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:18Z" name: success-200-isvc-f3231-predictor.18a968d40cd082c7 namespace: kserve-ci-e2e-test resourceVersion: "20455" uid: 532b08e2-b030-430c-9c88-4f1abe77a343 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:39:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f3231-predictor namespace: kserve-ci-e2e-test resourceVersion: "19916" uid: fcd80599-4396-4769-8fb5-1eb15173a371 kind: Event lastTimestamp: "2026-04-24T21:40:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T21:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:40:18Z" name: success-200-isvc-f3231-predictor.18a968d40d277904 namespace: kserve-ci-e2e-test resourceVersion: "20456" uid: 14e15824-a7ac-4240-a703-cb5900e04cf4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T21:39:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f3231 namespace: kserve-ci-e2e-test resourceVersion: "19675" uid: d33dacd4-7810-49c0-be24-8c6f906ff848 kind: Event lastTimestamp: "2026-04-24T21:40:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T21:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:40:05Z" name: success-200-isvc-f3231.18a968cd0de45dce namespace: kserve-ci-e2e-test resourceVersion: "20328" uid: 020e704c-87ca-4cce-bab7-55cab83bb5b5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f3231 namespace: kserve-ci-e2e-test resourceVersion: "19675" uid: d33dacd4-7810-49c0-be24-8c6f906ff848 kind: Event lastTimestamp: "2026-04-24T21:39:17Z" message: 'Failed to update status for InferenceService "success-200-isvc-f3231": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f3231": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:39:17Z" name: success-200-isvc-f3231.18a968cd0e3ee01d namespace: kserve-ci-e2e-test resourceVersion: "19703" uid: cc504362-9d85-4071-9c27-186060bfbf85 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f3231 namespace: kserve-ci-e2e-test resourceVersion: "19675" uid: d33dacd4-7810-49c0-be24-8c6f906ff848 kind: Event lastTimestamp: "2026-04-24T21:39:17Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f3231": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:39:17Z" name: success-200-isvc-f3231.18a968cd0e3f4209 namespace: kserve-ci-e2e-test resourceVersion: "19704" uid: 85f3a153-ac5a-4105-9657-3f742dae66dc reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:40:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f3231 namespace: kserve-ci-e2e-test resourceVersion: "20325" uid: d33dacd4-7810-49c0-be24-8c6f906ff848 kind: Event lastTimestamp: "2026-04-24T21:40:05Z" message: InferenceService [success-200-isvc-f3231] is Ready metadata: creationTimestamp: "2026-04-24T21:40:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:40:05Z" name: success-200-isvc-f3231.18a968d82d3d6af7 namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: 7517e869-8e75-4e53-b2e2-9f85a2897173 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:51:19.063670Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-98aac-579b965bb4-xcntc namespace: kserve-ci-e2e-test resourceVersion: "26457" uid: d1535524-2257-4d12-a251-848068a29cc7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-98aac-579b965bb4-xcntc to ip-10-0-128-21.ec2.internal metadata: creationTimestamp: "2026-04-24T21:51:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:51:19Z" name: switch-graph-98aac-579b965bb4-xcntc.18a96974f77d4b31 namespace: kserve-ci-e2e-test resourceVersion: "26462" uid: bb82580a-1d88-4187-a9a6-550702c5514f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:51:19Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-98aac-579b965bb4-xcntc namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: d1535524-2257-4d12-a251-848068a29cc7 kind: Event lastTimestamp: "2026-04-24T21:51:19Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-98aac-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:51:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:51:19Z" name: switch-graph-98aac-579b965bb4-xcntc.18a96975026a3e03 namespace: kserve-ci-e2e-test resourceVersion: "26479" uid: 080bca4b-794a-411b-bfc2-6252347a238a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:51:20Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-98aac-579b965bb4-xcntc namespace: kserve-ci-e2e-test resourceVersion: "26466" uid: d1535524-2257-4d12-a251-848068a29cc7 kind: Event lastTimestamp: "2026-04-24T21:51:20Z" message: Add eth0 [10.132.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:51:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:51:20Z" name: switch-graph-98aac-579b965bb4-xcntc.18a9697534b9f3a3 namespace: kserve-ci-e2e-test resourceVersion: "26491" uid: ebc03836-e733-4c73-b810-0503ddb42059 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:51:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-98aac} kind: Pod name: switch-graph-98aac-579b965bb4-xcntc namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: d1535524-2257-4d12-a251-848068a29cc7 kind: Event lastTimestamp: "2026-04-24T21:51:20Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f2021b9f79dbd151723ae4c386dd28bedd0fff23f6daa0825bdc58598fc71887" already present on machine metadata: creationTimestamp: "2026-04-24T21:51:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:51:20Z" name: switch-graph-98aac-579b965bb4-xcntc.18a969753607bef3 namespace: kserve-ci-e2e-test resourceVersion: "26493" uid: 4d9a4831-008d-42a9-aaea-08a950f67acc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:51:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-98aac} kind: Pod name: switch-graph-98aac-579b965bb4-xcntc namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: d1535524-2257-4d12-a251-848068a29cc7 kind: Event lastTimestamp: "2026-04-24T21:51:20Z" message: 'Created container: switch-graph-98aac' metadata: creationTimestamp: "2026-04-24T21:51:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:51:20Z" name: switch-graph-98aac-579b965bb4-xcntc.18a9697539031f27 namespace: kserve-ci-e2e-test resourceVersion: "26494" uid: f6e4a852-c4cb-4a2d-ab8e-4f965a890785 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:51:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-98aac} kind: Pod name: switch-graph-98aac-579b965bb4-xcntc namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: d1535524-2257-4d12-a251-848068a29cc7 kind: Event lastTimestamp: "2026-04-24T21:51:20Z" message: Started container switch-graph-98aac metadata: creationTimestamp: "2026-04-24T21:51:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:51:20Z" name: switch-graph-98aac-579b965bb4-xcntc.18a969753978fff0 namespace: kserve-ci-e2e-test resourceVersion: "26495" uid: 03cceaaf-5de6-4e89-95d9-24fa0d59cf0c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-98aac} kind: Pod name: switch-graph-98aac-579b965bb4-xcntc namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: d1535524-2257-4d12-a251-848068a29cc7 kind: Event lastTimestamp: "2026-04-24T22:07:38Z" message: Stopping container switch-graph-98aac metadata: creationTimestamp: "2026-04-24T22:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:07:38Z" name: switch-graph-98aac-579b965bb4-xcntc.18a96a58f9c8e05a namespace: kserve-ci-e2e-test resourceVersion: "33377" uid: 3a483cdc-8f19-4754-a676-91046fc86645 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T22:07:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-98aac} kind: Pod name: switch-graph-98aac-579b965bb4-xcntc namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: d1535524-2257-4d12-a251-848068a29cc7 kind: Event lastTimestamp: "2026-04-24T22:08:01Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T22:07:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:08:01Z" name: switch-graph-98aac-579b965bb4-xcntc.18a96a59d23d13e6 namespace: kserve-ci-e2e-test resourceVersion: "33618" uid: b6078575-88aa-4edb-8c86-0f9118a8c080 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:51:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-98aac-579b965bb4 namespace: kserve-ci-e2e-test resourceVersion: "26453" uid: a02f61bd-3d39-4068-81b0-1b54a1ddbcb3 kind: Event lastTimestamp: "2026-04-24T21:51:19Z" message: 'Created pod: switch-graph-98aac-579b965bb4-xcntc' metadata: creationTimestamp: "2026-04-24T21:51:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:51:19Z" name: switch-graph-98aac-579b965bb4.18a96974f720a6fb namespace: kserve-ci-e2e-test resourceVersion: "26459" uid: 7d1f0cab-bedb-4f52-9429-3e05c3042890 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:51:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-98aac namespace: kserve-ci-e2e-test resourceVersion: "26450" uid: 8ad5e9ce-de40-477b-ab8e-88b558eede0f kind: Event lastTimestamp: "2026-04-24T21:51:19Z" message: Scaled up replica set switch-graph-98aac-579b965bb4 from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:51:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:51:19Z" name: switch-graph-98aac.18a96974f682556c namespace: kserve-ci-e2e-test resourceVersion: "26455" uid: 46961c3c-43a1-404b-a895-515025806efd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:51:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-98aac namespace: kserve-ci-e2e-test resourceVersion: "26469" uid: 9519479b-0f89-4382-b84b-58d1aded32d2 kind: Event lastTimestamp: "2026-04-24T21:51:19Z" message: InferenceGraph [switch-graph-98aac] is Ready metadata: creationTimestamp: "2026-04-24T21:51:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:51:19Z" name: switch-graph-98aac.18a96974fa88ce97 namespace: kserve-ci-e2e-test resourceVersion: "26472" uid: 6bbc50dd-a660-47f5-b702-dda018d5173d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:51:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-98aac namespace: kserve-ci-e2e-test resourceVersion: "26443" uid: 9519479b-0f89-4382-b84b-58d1aded32d2 kind: Event lastTimestamp: "2026-04-24T21:51:19Z" message: 'Failed to update status for InferenceGraph "switch-graph-98aac": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-98aac": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:51:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:51:19Z" name: switch-graph-98aac.18a96974fca6a9a7 namespace: kserve-ci-e2e-test resourceVersion: "26474" uid: 2c0e314a-06ac-49fb-9e1e-3a420fd2e4d5 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:51:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-98aac namespace: kserve-ci-e2e-test resourceVersion: "26443" uid: 9519479b-0f89-4382-b84b-58d1aded32d2 kind: Event lastTimestamp: "2026-04-24T21:51:19Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-98aac": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:51:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:51:19Z" name: switch-graph-98aac.18a96974fca6fc66 namespace: kserve-ci-e2e-test resourceVersion: "26476" uid: e0111d5e-8d46-44a9-9396-de612b17cdd0 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:51:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-98aac namespace: kserve-ci-e2e-test resourceVersion: "26452" uid: f8bae093-a746-4b1d-bfd9-4e58a4d27990 kind: Event lastTimestamp: "2026-04-24T21:51:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:51:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:51:49Z" name: switch-graph-98aac.18a96978769de407 namespace: kserve-ci-e2e-test resourceVersion: "26743" uid: 87d2c56b-c589-4f1d-83fb-8ade984906fb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T21:51:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-98aac namespace: kserve-ci-e2e-test resourceVersion: "26452" uid: f8bae093-a746-4b1d-bfd9-4e58a4d27990 kind: Event lastTimestamp: "2026-04-24T21:51:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:51:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:51:49Z" name: switch-graph-98aac.18a9697877a28d8a namespace: kserve-ci-e2e-test resourceVersion: "26744" uid: 4746c90b-ae15-4ed5-a5f4-8040018ba07e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:31:03.095869Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-b868d-787648c54b-r6sn7 namespace: kserve-ci-e2e-test resourceVersion: "16145" uid: 7ea2ad55-ea51-4865-888c-7206e04f3c32 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-b868d-787648c54b-r6sn7 to ip-10-0-128-21.ec2.internal metadata: creationTimestamp: "2026-04-24T21:31:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:31:03Z" name: switch-graph-b868d-787648c54b-r6sn7.18a96859da29203c namespace: kserve-ci-e2e-test resourceVersion: "16150" uid: c975213c-1f03-49a2-9cdf-2bf02a82f742 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:31:03Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-b868d-787648c54b-r6sn7 namespace: kserve-ci-e2e-test resourceVersion: "16154" uid: 7ea2ad55-ea51-4865-888c-7206e04f3c32 kind: Event lastTimestamp: "2026-04-24T21:31:03Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:31:03Z" name: switch-graph-b868d-787648c54b-r6sn7.18a96859f4c3a546 namespace: kserve-ci-e2e-test resourceVersion: "16169" uid: 4fef87d6-df09-4cfc-a452-96a8f61565c0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:31:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b868d} kind: Pod name: switch-graph-b868d-787648c54b-r6sn7 namespace: kserve-ci-e2e-test resourceVersion: "16148" uid: 7ea2ad55-ea51-4865-888c-7206e04f3c32 kind: Event lastTimestamp: "2026-04-24T21:31:03Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f2021b9f79dbd151723ae4c386dd28bedd0fff23f6daa0825bdc58598fc71887" already present on machine metadata: creationTimestamp: "2026-04-24T21:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:31:03Z" name: switch-graph-b868d-787648c54b-r6sn7.18a96859f661b4ef namespace: kserve-ci-e2e-test resourceVersion: "16172" uid: 3a3bc064-10e6-4d56-b00a-6c50223f861b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:31:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b868d} kind: Pod name: switch-graph-b868d-787648c54b-r6sn7 namespace: kserve-ci-e2e-test resourceVersion: "16148" uid: 7ea2ad55-ea51-4865-888c-7206e04f3c32 kind: Event lastTimestamp: "2026-04-24T21:31:03Z" message: 'Created container: switch-graph-b868d' metadata: creationTimestamp: "2026-04-24T21:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:31:03Z" name: switch-graph-b868d-787648c54b-r6sn7.18a96859f9d3e674 namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: f9f8d27d-67ba-44ca-a15e-4cffb27b5e90 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:31:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b868d} kind: Pod name: switch-graph-b868d-787648c54b-r6sn7 namespace: kserve-ci-e2e-test resourceVersion: "16148" uid: 7ea2ad55-ea51-4865-888c-7206e04f3c32 kind: Event lastTimestamp: "2026-04-24T21:31:03Z" message: Started container switch-graph-b868d metadata: creationTimestamp: "2026-04-24T21:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:31:03Z" name: switch-graph-b868d-787648c54b-r6sn7.18a96859fa4a0649 namespace: kserve-ci-e2e-test resourceVersion: "16175" uid: 60758349-1da2-455d-92b0-e460d8f0899b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:39:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b868d} kind: Pod name: switch-graph-b868d-787648c54b-r6sn7 namespace: kserve-ci-e2e-test resourceVersion: "16148" uid: 7ea2ad55-ea51-4865-888c-7206e04f3c32 kind: Event lastTimestamp: "2026-04-24T21:39:17Z" message: Stopping container switch-graph-b868d metadata: creationTimestamp: "2026-04-24T21:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:17Z" name: switch-graph-b868d-787648c54b-r6sn7.18a968cd0673135a namespace: kserve-ci-e2e-test resourceVersion: "19671" uid: 07e3537e-7eb2-4f4b-84b7-6eb7d3e419d4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-b868d} kind: Pod name: switch-graph-b868d-787648c54b-r6sn7 namespace: kserve-ci-e2e-test resourceVersion: "16148" uid: 7ea2ad55-ea51-4865-888c-7206e04f3c32 kind: Event lastTimestamp: "2026-04-24T21:39:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:39:44Z" name: switch-graph-b868d-787648c54b-r6sn7.18a968cd8994e32c namespace: kserve-ci-e2e-test resourceVersion: "20007" uid: d17f1491-ec41-4947-900e-e7cdbd148437 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:31:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-b868d-787648c54b namespace: kserve-ci-e2e-test resourceVersion: "16142" uid: 86089cfa-c5b4-49ca-a392-d24ca5920a92 kind: Event lastTimestamp: "2026-04-24T21:31:03Z" message: 'Created pod: switch-graph-b868d-787648c54b-r6sn7' metadata: creationTimestamp: "2026-04-24T21:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:31:03Z" name: switch-graph-b868d-787648c54b.18a96859d9c8152f namespace: kserve-ci-e2e-test resourceVersion: "16147" uid: d921a573-6539-4953-b075-d97483e9bdf3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:31:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-b868d namespace: kserve-ci-e2e-test resourceVersion: "16139" uid: ae12ecfe-e84d-4760-9883-7e3ec8cc1437 kind: Event lastTimestamp: "2026-04-24T21:31:03Z" message: Scaled up replica set switch-graph-b868d-787648c54b from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:31:03Z" name: switch-graph-b868d.18a96859d8db2396 namespace: kserve-ci-e2e-test resourceVersion: "16143" uid: 3b75601c-06cf-49db-b2a8-6b66f208e81a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:31:03Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-b868d namespace: kserve-ci-e2e-test resourceVersion: "16156" uid: 018bdc22-9cdd-4d6a-b129-1e252d862028 kind: Event lastTimestamp: "2026-04-24T21:31:03Z" message: InferenceGraph [switch-graph-b868d] is Ready metadata: creationTimestamp: "2026-04-24T21:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:31:03Z" name: switch-graph-b868d.18a96859ddd46aab namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: 922ca50e-1c33-4615-916f-e73f5886104d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:31:03Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-b868d namespace: kserve-ci-e2e-test resourceVersion: "16130" uid: 018bdc22-9cdd-4d6a-b129-1e252d862028 kind: Event lastTimestamp: "2026-04-24T21:31:03Z" message: 'Failed to update status for InferenceGraph "switch-graph-b868d": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-b868d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:31:03Z" name: switch-graph-b868d.18a96859dffcd6ad namespace: kserve-ci-e2e-test resourceVersion: "16161" uid: ddf8f9a0-6568-4841-abcc-841761273465 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:31:03Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-b868d namespace: kserve-ci-e2e-test resourceVersion: "16130" uid: 018bdc22-9cdd-4d6a-b129-1e252d862028 kind: Event lastTimestamp: "2026-04-24T21:31:03Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-b868d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T21:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:31:03Z" name: switch-graph-b868d.18a96859dffd2660 namespace: kserve-ci-e2e-test resourceVersion: "16162" uid: a73c0051-03d5-45f5-81b3-50632f871d3c reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:31:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-b868d namespace: kserve-ci-e2e-test resourceVersion: "16141" uid: cb36ad78-fc22-4fd4-a8df-1b8acf04900a kind: Event lastTimestamp: "2026-04-24T21:31:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:31:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:31:48Z" name: switch-graph-b868d.18a9685d57363e9f namespace: kserve-ci-e2e-test resourceVersion: "16586" uid: 07a85bc8-9a9e-49c0-b5eb-5ce1a383a739 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T21:31:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-b868d namespace: kserve-ci-e2e-test resourceVersion: "16141" uid: cb36ad78-fc22-4fd4-a8df-1b8acf04900a kind: Event lastTimestamp: "2026-04-24T21:31:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T21:31:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:31:48Z" name: switch-graph-b868d.18a9685d57987cf5 namespace: kserve-ci-e2e-test resourceVersion: "16587" uid: 13c8c6d1-5c7d-4dd8-a1c2-658f4b2e3fee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T21:29:48.740476Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d41b1-655d9c9d4c-sgp2v namespace: kserve-ci-e2e-test resourceVersion: "15096" uid: 869d7610-9d15-4439-b9fb-dfe142a340d6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-d41b1-655d9c9d4c-sgp2v to ip-10-0-128-21.ec2.internal metadata: creationTimestamp: "2026-04-24T21:29:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T21:29:48Z" name: switch-graph-d41b1-655d9c9d4c-sgp2v.18a968488a3bd373 namespace: kserve-ci-e2e-test resourceVersion: "15102" uid: 446b7191-9dba-42e9-8508-665ab74afa12 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c45bf474c-kfmd4 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:48Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d41b1-655d9c9d4c-sgp2v namespace: kserve-ci-e2e-test resourceVersion: "15099" uid: 869d7610-9d15-4439-b9fb-dfe142a340d6 kind: Event lastTimestamp: "2026-04-24T21:29:48Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-d41b1-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T21:29:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:29:48Z" name: switch-graph-d41b1-655d9c9d4c-sgp2v.18a9684894a74477 namespace: kserve-ci-e2e-test resourceVersion: "15113" uid: 9d3e3ea1-542d-49fa-8e5e-d58d220be58d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:49Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d41b1-655d9c9d4c-sgp2v namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 869d7610-9d15-4439-b9fb-dfe142a340d6 kind: Event lastTimestamp: "2026-04-24T21:29:49Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T21:29:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T21:29:49Z" name: switch-graph-d41b1-655d9c9d4c-sgp2v.18a96848c6fb7144 namespace: kserve-ci-e2e-test resourceVersion: "15123" uid: 7c0f7a4b-8706-4a8c-9545-f80ca695b5bd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d41b1} kind: Pod name: switch-graph-d41b1-655d9c9d4c-sgp2v namespace: kserve-ci-e2e-test resourceVersion: "15099" uid: 869d7610-9d15-4439-b9fb-dfe142a340d6 kind: Event lastTimestamp: "2026-04-24T21:29:49Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:f2021b9f79dbd151723ae4c386dd28bedd0fff23f6daa0825bdc58598fc71887" metadata: creationTimestamp: "2026-04-24T21:29:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:29:49Z" name: switch-graph-d41b1-655d9c9d4c-sgp2v.18a96848c852effe namespace: kserve-ci-e2e-test resourceVersion: "15125" uid: 166b972e-b43e-4ca6-855e-25c0b014134e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d41b1} kind: Pod name: switch-graph-d41b1-655d9c9d4c-sgp2v namespace: kserve-ci-e2e-test resourceVersion: "15099" uid: 869d7610-9d15-4439-b9fb-dfe142a340d6 kind: Event lastTimestamp: "2026-04-24T21:29:51Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:f2021b9f79dbd151723ae4c386dd28bedd0fff23f6daa0825bdc58598fc71887" in 1.995s (1.995s including waiting). Image size: 216340060 bytes.' metadata: creationTimestamp: "2026-04-24T21:29:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:29:51Z" name: switch-graph-d41b1-655d9c9d4c-sgp2v.18a968493f43207e namespace: kserve-ci-e2e-test resourceVersion: "15138" uid: 2994fe14-c883-4420-9504-f29376305c00 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d41b1} kind: Pod name: switch-graph-d41b1-655d9c9d4c-sgp2v namespace: kserve-ci-e2e-test resourceVersion: "15099" uid: 869d7610-9d15-4439-b9fb-dfe142a340d6 kind: Event lastTimestamp: "2026-04-24T21:29:51Z" message: 'Created container: switch-graph-d41b1' metadata: creationTimestamp: "2026-04-24T21:29:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:29:51Z" name: switch-graph-d41b1-655d9c9d4c-sgp2v.18a96849421ddad7 namespace: kserve-ci-e2e-test resourceVersion: "15139" uid: 663d1488-a7c2-4b2d-bd93-2f5531c6b484 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d41b1} kind: Pod name: switch-graph-d41b1-655d9c9d4c-sgp2v namespace: kserve-ci-e2e-test resourceVersion: "15099" uid: 869d7610-9d15-4439-b9fb-dfe142a340d6 kind: Event lastTimestamp: "2026-04-24T21:29:51Z" message: Started container switch-graph-d41b1 metadata: creationTimestamp: "2026-04-24T21:29:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:29:51Z" name: switch-graph-d41b1-655d9c9d4c-sgp2v.18a9684942938453 namespace: kserve-ci-e2e-test resourceVersion: "15140" uid: f3570e70-7921-4ada-983f-542cbef3e8b9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:30:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d41b1} kind: Pod name: switch-graph-d41b1-655d9c9d4c-sgp2v namespace: kserve-ci-e2e-test resourceVersion: "15099" uid: 869d7610-9d15-4439-b9fb-dfe142a340d6 kind: Event lastTimestamp: "2026-04-24T21:30:02Z" message: Stopping container switch-graph-d41b1 metadata: creationTimestamp: "2026-04-24T21:30:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:02Z" name: switch-graph-d41b1-655d9c9d4c-sgp2v.18a9684bd36d15b3 namespace: kserve-ci-e2e-test resourceVersion: "15275" uid: 82202ffc-3466-46a5-87fe-a79892c2e2bf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T21:30:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d41b1} kind: Pod name: switch-graph-d41b1-655d9c9d4c-sgp2v namespace: kserve-ci-e2e-test resourceVersion: "15099" uid: 869d7610-9d15-4439-b9fb-dfe142a340d6 kind: Event lastTimestamp: "2026-04-24T21:30:28Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T21:30:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T21:30:28Z" name: switch-graph-d41b1-655d9c9d4c-sgp2v.18a9684c06b380f4 namespace: kserve-ci-e2e-test resourceVersion: "15622" uid: 7c33eeff-2130-45d1-a72b-880a4f0b26ff reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-21.ec2.internal source: component: kubelet host: ip-10-0-128-21.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-d41b1-655d9c9d4c namespace: kserve-ci-e2e-test resourceVersion: "15093" uid: 1a83e6ef-8120-4888-985e-7965563aa35a kind: Event lastTimestamp: "2026-04-24T21:29:48Z" message: 'Created pod: switch-graph-d41b1-655d9c9d4c-sgp2v' metadata: creationTimestamp: "2026-04-24T21:29:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:29:48Z" name: switch-graph-d41b1-655d9c9d4c.18a9684889ac1259 namespace: kserve-ci-e2e-test resourceVersion: "15098" uid: ebe24787-78a7-4d6a-915d-e7a95b210835 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-d41b1 namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: 17fea074-2c6e-45b9-a10a-ff97aca8104d kind: Event lastTimestamp: "2026-04-24T21:29:48Z" message: Scaled up replica set switch-graph-d41b1-655d9c9d4c from 0 to 1 metadata: creationTimestamp: "2026-04-24T21:29:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T21:29:48Z" name: switch-graph-d41b1.18a9684888c45735 namespace: kserve-ci-e2e-test resourceVersion: "15094" uid: dca7f165-d133-4897-9edc-71b608976af3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T21:29:48Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-d41b1 namespace: kserve-ci-e2e-test resourceVersion: "15107" uid: 0392a077-f3f2-4ea3-976a-8d6a7832bf8a kind: Event lastTimestamp: "2026-04-24T21:29:48Z" message: InferenceGraph [switch-graph-d41b1] is Ready metadata: creationTimestamp: "2026-04-24T21:29:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T21:29:48Z" name: switch-graph-d41b1.18a968488c683533 namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 9f266c11-e866-4f62-abeb-cefaa6f5ffe7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal kind: EventList metadata: resourceVersion: "33628"