--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:59:43.969347Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-af621-7d4fd998f4-sv5l4 namespace: kserve-ci-e2e-test resourceVersion: "21189" uid: ba5c9b1a-46ff-4794-985b-6e2a883239e0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-af621-7d4fd998f4-sv5l4 to ip-10-0-139-178.ec2.internal metadata: creationTimestamp: "2026-07-01T18:59:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:59:43Z" name: ensemble-graph-af621-7d4fd998f4-sv5l4.18be3f8e8d08a35a namespace: kserve-ci-e2e-test resourceVersion: "21196" uid: b9a7e34a-7960-4b8d-ad1c-68c23c42aab8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:44Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-af621-7d4fd998f4-sv5l4 namespace: kserve-ci-e2e-test resourceVersion: "21200" uid: ba5c9b1a-46ff-4794-985b-6e2a883239e0 kind: Event lastTimestamp: "2026-07-01T18:59:44Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:59:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:59:44Z" name: ensemble-graph-af621-7d4fd998f4-sv5l4.18be3f8ea6e548af namespace: kserve-ci-e2e-test resourceVersion: "21213" uid: c316cad0-c0e9-4da3-8361-0bd20562c819 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-af621} kind: Pod name: ensemble-graph-af621-7d4fd998f4-sv5l4 namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: ba5c9b1a-46ff-4794-985b-6e2a883239e0 kind: Event lastTimestamp: "2026-07-01T18:59:44Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a589cb7fd67a31c767ba2472b47f816cadffddbd6c2e2c8ddfd876f9a79d8448" already present on machine metadata: creationTimestamp: "2026-07-01T18:59:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:44Z" name: ensemble-graph-af621-7d4fd998f4-sv5l4.18be3f8ea84b88b9 namespace: kserve-ci-e2e-test resourceVersion: "21215" uid: 5d1a0005-050f-49fe-b335-3130f05a68e9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-af621} kind: Pod name: ensemble-graph-af621-7d4fd998f4-sv5l4 namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: ba5c9b1a-46ff-4794-985b-6e2a883239e0 kind: Event lastTimestamp: "2026-07-01T18:59:44Z" message: 'Created container: ensemble-graph-af621' metadata: creationTimestamp: "2026-07-01T18:59:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:44Z" name: ensemble-graph-af621-7d4fd998f4-sv5l4.18be3f8eab3387d2 namespace: kserve-ci-e2e-test resourceVersion: "21216" uid: bc5c525a-eb61-4596-884d-a9397ff393fb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-af621} kind: Pod name: ensemble-graph-af621-7d4fd998f4-sv5l4 namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: ba5c9b1a-46ff-4794-985b-6e2a883239e0 kind: Event lastTimestamp: "2026-07-01T18:59:44Z" message: Started container ensemble-graph-af621 metadata: creationTimestamp: "2026-07-01T18:59:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:44Z" name: ensemble-graph-af621-7d4fd998f4-sv5l4.18be3f8eaba5ca57 namespace: kserve-ci-e2e-test resourceVersion: "21217" uid: e519317e-b91f-4aba-b9f6-5ab36ad20d1f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:07:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-af621} kind: Pod name: ensemble-graph-af621-7d4fd998f4-sv5l4 namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: ba5c9b1a-46ff-4794-985b-6e2a883239e0 kind: Event lastTimestamp: "2026-07-01T19:07:58Z" message: Stopping container ensemble-graph-af621 metadata: creationTimestamp: "2026-07-01T19:07:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:07:58Z" name: ensemble-graph-af621-7d4fd998f4-sv5l4.18be4001bd897714 namespace: kserve-ci-e2e-test resourceVersion: "24921" uid: b783f1db-c91a-40a0-b9fd-a05f7dd3ad5d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:08:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-af621} kind: Pod name: ensemble-graph-af621-7d4fd998f4-sv5l4 namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: ba5c9b1a-46ff-4794-985b-6e2a883239e0 kind: Event lastTimestamp: "2026-07-01T19:08:25Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T19:08:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:08:25Z" name: ensemble-graph-af621-7d4fd998f4-sv5l4.18be4002333496b0 namespace: kserve-ci-e2e-test resourceVersion: "25252" uid: ac7174ea-3b5b-49b7-a14c-89b3d3bf1e8b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-af621-7d4fd998f4 namespace: kserve-ci-e2e-test resourceVersion: "21186" uid: 00bf9ee4-b354-484d-bf23-42439ced7042 kind: Event lastTimestamp: "2026-07-01T18:59:43Z" message: 'Created pod: ensemble-graph-af621-7d4fd998f4-sv5l4' metadata: creationTimestamp: "2026-07-01T18:59:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:59:43Z" name: ensemble-graph-af621-7d4fd998f4.18be3f8e8c6d9fcb namespace: kserve-ci-e2e-test resourceVersion: "21191" uid: 54d78d03-6b90-4563-8b9d-1382e3289b58 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-af621 namespace: kserve-ci-e2e-test resourceVersion: "21184" uid: 36f0a5f6-ab3d-4591-bc12-f7fef39325ad kind: Event lastTimestamp: "2026-07-01T18:59:43Z" message: Scaled up replica set ensemble-graph-af621-7d4fd998f4 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:59:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:59:43Z" name: ensemble-graph-af621.18be3f8e8b9c05ad namespace: kserve-ci-e2e-test resourceVersion: "21187" uid: 626973a9-9069-4a6a-994c-5a380e740b30 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:44Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-af621 namespace: kserve-ci-e2e-test resourceVersion: "21203" uid: cd9fefaf-70c0-4920-8bb9-5666e378689e kind: Event lastTimestamp: "2026-07-01T18:59:44Z" message: InferenceGraph [ensemble-graph-af621] is Ready metadata: creationTimestamp: "2026-07-01T18:59:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:59:44Z" name: ensemble-graph-af621.18be3f8e9156d619 namespace: kserve-ci-e2e-test resourceVersion: "21206" uid: 722facb6-97ba-41a5-abf7-8268ae305a70 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:59:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-af621 namespace: kserve-ci-e2e-test resourceVersion: "21185" uid: 4cba2b2f-639b-4d87-8cc4-ededea868730 kind: Event lastTimestamp: "2026-07-01T19:00:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:00:13Z" name: ensemble-graph-af621.18be3f920aec4fc4 namespace: kserve-ci-e2e-test resourceVersion: "21662" uid: ce8043f6-b55a-4c5f-a076-a738f50d4b0b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:59:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-af621 namespace: kserve-ci-e2e-test resourceVersion: "21185" uid: 4cba2b2f-639b-4d87-8cc4-ededea868730 kind: Event lastTimestamp: "2026-07-01T19:00:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:00:14Z" name: ensemble-graph-af621.18be3f920b785e3e namespace: kserve-ci-e2e-test resourceVersion: "21663" uid: 1f3ccaaf-39a1-4f16-8942-a3589bce75d0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:58:33.680389Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c059b-b4df54dc8-fppfl namespace: kserve-ci-e2e-test resourceVersion: "20374" uid: e8ecbd2a-6ea7-41de-850a-369c27f35b9a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-c059b-b4df54dc8-fppfl to ip-10-0-139-178.ec2.internal metadata: creationTimestamp: "2026-07-01T18:58:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:58:33Z" name: ensemble-graph-c059b-b4df54dc8-fppfl.18be3f7e2f7c3972 namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: 1fc8dac9-cab5-4697-af87-eb4ef7c5c08f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:34Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c059b-b4df54dc8-fppfl namespace: kserve-ci-e2e-test resourceVersion: "20383" uid: e8ecbd2a-6ea7-41de-850a-369c27f35b9a kind: Event lastTimestamp: "2026-07-01T18:58:34Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:58:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:58:34Z" name: ensemble-graph-c059b-b4df54dc8-fppfl.18be3f7e48f6a64b namespace: kserve-ci-e2e-test resourceVersion: "20398" uid: af70dfbd-a9c6-4864-9f82-f84c40974221 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c059b} kind: Pod name: ensemble-graph-c059b-b4df54dc8-fppfl namespace: kserve-ci-e2e-test resourceVersion: "20375" uid: e8ecbd2a-6ea7-41de-850a-369c27f35b9a kind: Event lastTimestamp: "2026-07-01T18:58:34Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a589cb7fd67a31c767ba2472b47f816cadffddbd6c2e2c8ddfd876f9a79d8448" already present on machine metadata: creationTimestamp: "2026-07-01T18:58:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:34Z" name: ensemble-graph-c059b-b4df54dc8-fppfl.18be3f7e4a4dbba7 namespace: kserve-ci-e2e-test resourceVersion: "20400" uid: 8b0deff7-cd43-482c-b891-0376a8c7e331 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c059b} kind: Pod name: ensemble-graph-c059b-b4df54dc8-fppfl namespace: kserve-ci-e2e-test resourceVersion: "20375" uid: e8ecbd2a-6ea7-41de-850a-369c27f35b9a kind: Event lastTimestamp: "2026-07-01T18:58:34Z" message: 'Created container: ensemble-graph-c059b' metadata: creationTimestamp: "2026-07-01T18:58:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:34Z" name: ensemble-graph-c059b-b4df54dc8-fppfl.18be3f7e4d27749f namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 9293469b-d1a8-4e0f-87c3-08887f227c33 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c059b} kind: Pod name: ensemble-graph-c059b-b4df54dc8-fppfl namespace: kserve-ci-e2e-test resourceVersion: "20375" uid: e8ecbd2a-6ea7-41de-850a-369c27f35b9a kind: Event lastTimestamp: "2026-07-01T18:58:34Z" message: Started container ensemble-graph-c059b metadata: creationTimestamp: "2026-07-01T18:58:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:34Z" name: ensemble-graph-c059b-b4df54dc8-fppfl.18be3f7e4db50a74 namespace: kserve-ci-e2e-test resourceVersion: "20403" uid: 3c2d91a0-9a60-4a54-a8a6-12ea98ac9b92 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c059b} kind: Pod name: ensemble-graph-c059b-b4df54dc8-fppfl namespace: kserve-ci-e2e-test resourceVersion: "20375" uid: e8ecbd2a-6ea7-41de-850a-369c27f35b9a kind: Event lastTimestamp: "2026-07-01T18:58:43Z" message: Stopping container ensemble-graph-c059b metadata: creationTimestamp: "2026-07-01T18:58:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:43Z" name: ensemble-graph-c059b-b4df54dc8-fppfl.18be3f808780632e namespace: kserve-ci-e2e-test resourceVersion: "20499" uid: e13505e1-b9f1-4ff2-adc4-bd108909ac01 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:58:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c059b} kind: Pod name: ensemble-graph-c059b-b4df54dc8-fppfl namespace: kserve-ci-e2e-test resourceVersion: "20375" uid: e8ecbd2a-6ea7-41de-850a-369c27f35b9a kind: Event lastTimestamp: "2026-07-01T18:59:10Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T18:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:10Z" name: ensemble-graph-c059b-b4df54dc8-fppfl.18be3f80e71fcbba namespace: kserve-ci-e2e-test resourceVersion: "20855" uid: fe7d6ca8-e994-490e-a524-8612a98fa251 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-c059b-b4df54dc8 namespace: kserve-ci-e2e-test resourceVersion: "20371" uid: be334336-d672-43bb-9687-1e03fd8919fb kind: Event lastTimestamp: "2026-07-01T18:58:33Z" message: 'Created pod: ensemble-graph-c059b-b4df54dc8-fppfl' metadata: creationTimestamp: "2026-07-01T18:58:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:58:33Z" name: ensemble-graph-c059b-b4df54dc8.18be3f7e2ee6bdfe namespace: kserve-ci-e2e-test resourceVersion: "20376" uid: fec73329-e51f-4e09-99d6-c63e8ef3b773 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-c059b namespace: kserve-ci-e2e-test resourceVersion: "20369" uid: d93c6f52-0e97-49fe-9ad5-e24272665a76 kind: Event lastTimestamp: "2026-07-01T18:58:33Z" message: Scaled up replica set ensemble-graph-c059b-b4df54dc8 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:58:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:58:33Z" name: ensemble-graph-c059b.18be3f7e2dec4ec4 namespace: kserve-ci-e2e-test resourceVersion: "20372" uid: 7a5183b5-b7de-4c92-b664-1a074fd9e73f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:33Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-c059b namespace: kserve-ci-e2e-test resourceVersion: "20388" uid: 6b1b5a0e-324f-4f09-9a64-b34bd4bcd3a0 kind: Event lastTimestamp: "2026-07-01T18:58:33Z" message: InferenceGraph [ensemble-graph-c059b] is Ready metadata: creationTimestamp: "2026-07-01T18:58:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:58:33Z" name: ensemble-graph-c059b.18be3f7e341c1d5d namespace: kserve-ci-e2e-test resourceVersion: "20391" uid: 4945bf57-a645-49cd-b7a7-3db6c87239d6 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:07:59.016725Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5 namespace: kserve-ci-e2e-test resourceVersion: "24987" uid: a23e601c-df80-4e65-9d8e-a63cf7bbbc1c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5 to ip-10-0-129-245.ec2.internal metadata: creationTimestamp: "2026-07-01T19:07:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:07:59Z" name: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5.18be4001d028334e namespace: kserve-ci-e2e-test resourceVersion: "24991" uid: 2abd88a9-a13c-4c8e-a081-c0a5d3dcdd19 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:07:59Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5 namespace: kserve-ci-e2e-test resourceVersion: "24989" uid: a23e601c-df80-4e65-9d8e-a63cf7bbbc1c kind: Event lastTimestamp: "2026-07-01T19:07:59Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-1bf2c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:07:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:07:59Z" name: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5.18be4001d9fe2aac namespace: kserve-ci-e2e-test resourceVersion: "25017" uid: 5a27ac74-d6c5-4ef8-9be2-82be395b5faa reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:08:00Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5 namespace: kserve-ci-e2e-test resourceVersion: "25005" uid: a23e601c-df80-4e65-9d8e-a63cf7bbbc1c kind: Event lastTimestamp: "2026-07-01T19:08:00Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:08:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:08:00Z" name: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5.18be40020d92b8b7 namespace: kserve-ci-e2e-test resourceVersion: "25036" uid: e2a77c01-a246-43e9-9e38-53471a5eff32 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:08:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5 namespace: kserve-ci-e2e-test resourceVersion: "24989" uid: a23e601c-df80-4e65-9d8e-a63cf7bbbc1c kind: Event lastTimestamp: "2026-07-01T19:08:00Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T19:08:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:08:00Z" name: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5.18be40020ef6d38e namespace: kserve-ci-e2e-test resourceVersion: "25038" uid: 4ac35682-7882-44e4-a977-06ea0258f966 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:08:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5 namespace: kserve-ci-e2e-test resourceVersion: "24989" uid: a23e601c-df80-4e65-9d8e-a63cf7bbbc1c kind: Event lastTimestamp: "2026-07-01T19:08:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:08:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:08:00Z" name: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5.18be4002123b3b35 namespace: kserve-ci-e2e-test resourceVersion: "25039" uid: 8a953a2a-f5b9-47e8-8819-a795b92e5817 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:08:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5 namespace: kserve-ci-e2e-test resourceVersion: "24989" uid: a23e601c-df80-4e65-9d8e-a63cf7bbbc1c kind: Event lastTimestamp: "2026-07-01T19:08:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:08:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:08:00Z" name: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5.18be400212ffd133 namespace: kserve-ci-e2e-test resourceVersion: "25040" uid: d8fea1d3-c3ba-488c-b7ac-19749d159fa2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:08:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5 namespace: kserve-ci-e2e-test resourceVersion: "24989" uid: a23e601c-df80-4e65-9d8e-a63cf7bbbc1c kind: Event lastTimestamp: "2026-07-01T19:08:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:08:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:08:00Z" name: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5.18be4002130c80b1 namespace: kserve-ci-e2e-test resourceVersion: "25044" uid: ba45ae3f-51b8-46e8-9ed5-2d9c4ded9de3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:08:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5 namespace: kserve-ci-e2e-test resourceVersion: "24989" uid: a23e601c-df80-4e65-9d8e-a63cf7bbbc1c kind: Event lastTimestamp: "2026-07-01T19:08:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:08:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:08:00Z" name: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5.18be400215e27fe7 namespace: kserve-ci-e2e-test resourceVersion: "25045" uid: a9be55ce-858d-4c9f-8d09-92b193c66cfd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:08:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5 namespace: kserve-ci-e2e-test resourceVersion: "24989" uid: a23e601c-df80-4e65-9d8e-a63cf7bbbc1c kind: Event lastTimestamp: "2026-07-01T19:08:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:08:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:08:00Z" name: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5.18be40021653a3de namespace: kserve-ci-e2e-test resourceVersion: "25046" uid: 1ea9ac95-7fcc-4ad9-a2fd-ec36b348b484 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:08:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5 namespace: kserve-ci-e2e-test resourceVersion: "24989" uid: a23e601c-df80-4e65-9d8e-a63cf7bbbc1c kind: Event lastTimestamp: "2026-07-01T19:08:36Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:08:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:08:36Z" name: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5.18be40022f122cf1 namespace: kserve-ci-e2e-test resourceVersion: "25344" uid: 030988dd-21cb-4332-9f07-fb4d23a6f5b2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5 namespace: kserve-ci-e2e-test resourceVersion: "24989" uid: a23e601c-df80-4e65-9d8e-a63cf7bbbc1c kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:09Z" name: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5.18be4012219ca6b9 namespace: kserve-ci-e2e-test resourceVersion: "26068" uid: 20c69d20-8dc1-41ad-8df4-c8b9a7c32e2d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5 namespace: kserve-ci-e2e-test resourceVersion: "24989" uid: a23e601c-df80-4e65-9d8e-a63cf7bbbc1c kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:09Z" name: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5.18be4012219dff1f namespace: kserve-ci-e2e-test resourceVersion: "26069" uid: 9e54ccc4-19fe-45ef-9069-9c1e0d57d9ec reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5 namespace: kserve-ci-e2e-test resourceVersion: "24989" uid: a23e601c-df80-4e65-9d8e-a63cf7bbbc1c kind: Event lastTimestamp: "2026-07-01T19:09:11Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:09:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:11Z" name: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5.18be4012b73a2cca namespace: kserve-ci-e2e-test resourceVersion: "26218" uid: b81a02bd-1ee3-44ed-869e-0551c7110e0a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:07:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1bf2c-predictor-6c64b6bb44 namespace: kserve-ci-e2e-test resourceVersion: "24976" uid: 7be73bca-9c71-463e-8138-7c15305afe9e kind: Event lastTimestamp: "2026-07-01T19:07:59Z" message: 'Created pod: error-404-isvc-1bf2c-predictor-6c64b6bb44-scdj5' metadata: creationTimestamp: "2026-07-01T19:07:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:07:59Z" name: error-404-isvc-1bf2c-predictor-6c64b6bb44.18be4001cfb402dc namespace: kserve-ci-e2e-test resourceVersion: "24988" uid: 79ee9a9d-1268-4892-b716-584c60470ca5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:07:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1bf2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24973" uid: 2e033fa6-b63e-4235-9a5d-678c9075be14 kind: Event lastTimestamp: "2026-07-01T19:07:58Z" message: Scaled up replica set error-404-isvc-1bf2c-predictor-6c64b6bb44 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:07:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:07:58Z" name: error-404-isvc-1bf2c-predictor.18be4001cdb85224 namespace: kserve-ci-e2e-test resourceVersion: "24979" uid: 8cf9d5d5-5738-4de3-926f-77bbdd2fce85 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:08:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1bf2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24975" uid: 6ff51b00-3a74-481e-97ff-8d7c4e02c6c6 kind: Event lastTimestamp: "2026-07-01T19:08:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:08:13Z" name: error-404-isvc-1bf2c-predictor.18be40054c73ca13 namespace: kserve-ci-e2e-test resourceVersion: "25167" uid: 36b48ee0-7c61-40f2-923b-ef2651b7ae6f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:08:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1bf2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24975" uid: 6ff51b00-3a74-481e-97ff-8d7c4e02c6c6 kind: Event lastTimestamp: "2026-07-01T19:08:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:08:13Z" name: error-404-isvc-1bf2c-predictor.18be40054cdb0068 namespace: kserve-ci-e2e-test resourceVersion: "25169" uid: 38032405-11b4-412e-9c53-5a96808fa490 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:08:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1bf2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25168" uid: 6ff51b00-3a74-481e-97ff-8d7c4e02c6c6 kind: Event lastTimestamp: "2026-07-01T19:08:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:08:59Z" name: error-404-isvc-1bf2c-predictor.18be4008cb9e9ffe namespace: kserve-ci-e2e-test resourceVersion: "25545" uid: ef7adabe-343e-4add-91b9-4c336571c059 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:08:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1bf2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25168" uid: 6ff51b00-3a74-481e-97ff-8d7c4e02c6c6 kind: Event lastTimestamp: "2026-07-01T19:08:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:08:59Z" name: error-404-isvc-1bf2c-predictor.18be4008cc1f54e6 namespace: kserve-ci-e2e-test resourceVersion: "25546" uid: 33ce85a3-f5d4-4ebd-a30a-c2527b2121d1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:07:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1bf2c namespace: kserve-ci-e2e-test resourceVersion: "24942" uid: 1f3f7ff9-357e-471a-b075-a730946f8e33 kind: Event lastTimestamp: "2026-07-01T19:08:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:07:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:08:46Z" name: error-404-isvc-1bf2c.18be4001ce2b25fc namespace: kserve-ci-e2e-test resourceVersion: "25433" uid: a2d10c02-4501-4547-90b4-5bcf2656071a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:07:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1bf2c namespace: kserve-ci-e2e-test resourceVersion: "24942" uid: 1f3f7ff9-357e-471a-b075-a730946f8e33 kind: Event lastTimestamp: "2026-07-01T19:07:58Z" message: 'Failed to update status for InferenceService "error-404-isvc-1bf2c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1bf2c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:07:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:07:58Z" name: error-404-isvc-1bf2c.18be4001ceaef6a9 namespace: kserve-ci-e2e-test resourceVersion: "24983" uid: a79dde8d-3f41-4a96-81ac-0b02b541c17a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:07:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1bf2c namespace: kserve-ci-e2e-test resourceVersion: "24942" uid: 1f3f7ff9-357e-471a-b075-a730946f8e33 kind: Event lastTimestamp: "2026-07-01T19:07:58Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1bf2c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:07:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:07:58Z" name: error-404-isvc-1bf2c.18be4001ceaf5590 namespace: kserve-ci-e2e-test resourceVersion: "24986" uid: 615e12a0-8536-426e-974a-b872e4452b61 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:08:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1bf2c namespace: kserve-ci-e2e-test resourceVersion: "25430" uid: 1f3f7ff9-357e-471a-b075-a730946f8e33 kind: Event lastTimestamp: "2026-07-01T19:08:46Z" message: InferenceService [error-404-isvc-1bf2c] is Ready metadata: creationTimestamp: "2026-07-01T19:08:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:08:46Z" name: error-404-isvc-1bf2c.18be400cee6e1451 namespace: kserve-ci-e2e-test resourceVersion: "25431" uid: 5b7cb0b5-689b-4e8d-ab40-17fc34d23a55 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:09:08.746083Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-43223-predictor-5968f7b98d-s44nj namespace: kserve-ci-e2e-test resourceVersion: "26009" uid: 45c55b1a-7c32-4a6c-8b7a-f83bae151d1a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-43223-predictor-5968f7b98d-s44nj to ip-10-0-129-245.ec2.internal metadata: creationTimestamp: "2026-07-01T19:09:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:09:08Z" name: error-404-isvc-43223-predictor-5968f7b98d-s44nj.18be40120c59be8e namespace: kserve-ci-e2e-test resourceVersion: "26013" uid: 03865602-c066-46ea-b723-cfda84b4765f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:08Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-43223-predictor-5968f7b98d-s44nj namespace: kserve-ci-e2e-test resourceVersion: "26011" uid: 45c55b1a-7c32-4a6c-8b7a-f83bae151d1a kind: Event lastTimestamp: "2026-07-01T19:09:08Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-43223-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:09:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:08Z" name: error-404-isvc-43223-predictor-5968f7b98d-s44nj.18be401216bacb79 namespace: kserve-ci-e2e-test resourceVersion: "26024" uid: fd86df10-0443-481d-9ed8-05c950c5135f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-43223-predictor-5968f7b98d-s44nj namespace: kserve-ci-e2e-test resourceVersion: "26017" uid: 45c55b1a-7c32-4a6c-8b7a-f83bae151d1a kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:09:09Z" name: error-404-isvc-43223-predictor-5968f7b98d-s44nj.18be40124a5f9ff4 namespace: kserve-ci-e2e-test resourceVersion: "26161" uid: f4c9aa3f-aba0-4f53-9fc3-5310033a51dc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-43223-predictor-5968f7b98d-s44nj namespace: kserve-ci-e2e-test resourceVersion: "26011" uid: 45c55b1a-7c32-4a6c-8b7a-f83bae151d1a kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:09Z" name: error-404-isvc-43223-predictor-5968f7b98d-s44nj.18be40124bbceba4 namespace: kserve-ci-e2e-test resourceVersion: "26163" uid: e4c30f94-5fe7-4a7a-9ead-4639808ac355 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-43223-predictor-5968f7b98d-s44nj namespace: kserve-ci-e2e-test resourceVersion: "26011" uid: 45c55b1a-7c32-4a6c-8b7a-f83bae151d1a kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:09Z" name: error-404-isvc-43223-predictor-5968f7b98d-s44nj.18be40124ecbad90 namespace: kserve-ci-e2e-test resourceVersion: "26166" uid: 4545a2ff-a091-4880-94f0-9b26c18de6db reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-43223-predictor-5968f7b98d-s44nj namespace: kserve-ci-e2e-test resourceVersion: "26011" uid: 45c55b1a-7c32-4a6c-8b7a-f83bae151d1a kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:09Z" name: error-404-isvc-43223-predictor-5968f7b98d-s44nj.18be40124f38e8ab namespace: kserve-ci-e2e-test resourceVersion: "26167" uid: 58a51c70-96fd-4b52-8be7-864de29001cb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-43223-predictor-5968f7b98d-s44nj namespace: kserve-ci-e2e-test resourceVersion: "26011" uid: 45c55b1a-7c32-4a6c-8b7a-f83bae151d1a kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:09Z" name: error-404-isvc-43223-predictor-5968f7b98d-s44nj.18be40124f41d9ec namespace: kserve-ci-e2e-test resourceVersion: "26168" uid: 897caa26-0648-4f2a-83f9-385eaa8e7eff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-43223-predictor-5968f7b98d-s44nj namespace: kserve-ci-e2e-test resourceVersion: "26011" uid: 45c55b1a-7c32-4a6c-8b7a-f83bae151d1a kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:09Z" name: error-404-isvc-43223-predictor-5968f7b98d-s44nj.18be401251fc90f5 namespace: kserve-ci-e2e-test resourceVersion: "26173" uid: 250fc41c-0961-4c1e-bb1f-23af042a695a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-43223-predictor-5968f7b98d-s44nj namespace: kserve-ci-e2e-test resourceVersion: "26011" uid: 45c55b1a-7c32-4a6c-8b7a-f83bae151d1a kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:09Z" name: error-404-isvc-43223-predictor-5968f7b98d-s44nj.18be40125268163e namespace: kserve-ci-e2e-test resourceVersion: "26174" uid: a183a96a-c56e-4cdd-acae-ae9e1103c6f6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-43223-predictor-5968f7b98d-s44nj namespace: kserve-ci-e2e-test resourceVersion: "26011" uid: 45c55b1a-7c32-4a6c-8b7a-f83bae151d1a kind: Event lastTimestamp: "2026-07-01T19:09:46Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:46Z" name: error-404-isvc-43223-predictor-5968f7b98d-s44nj.18be40128c57e80a namespace: kserve-ci-e2e-test resourceVersion: "26536" uid: 478e60dc-a928-41bd-bf25-d4ff0031a46b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:26:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-43223-predictor-5968f7b98d-s44nj namespace: kserve-ci-e2e-test resourceVersion: "26011" uid: 45c55b1a-7c32-4a6c-8b7a-f83bae151d1a kind: Event lastTimestamp: "2026-07-01T19:26:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:26:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:26:38Z" name: error-404-isvc-43223-predictor-5968f7b98d-s44nj.18be4106693f158c namespace: kserve-ci-e2e-test resourceVersion: "33710" uid: cd94abbb-faad-41bb-b114-c7b3ee8f7dec reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:26:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-43223-predictor-5968f7b98d-s44nj namespace: kserve-ci-e2e-test resourceVersion: "26011" uid: 45c55b1a-7c32-4a6c-8b7a-f83bae151d1a kind: Event lastTimestamp: "2026-07-01T19:26:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:26:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:26:38Z" name: error-404-isvc-43223-predictor-5968f7b98d-s44nj.18be410669406e8e namespace: kserve-ci-e2e-test resourceVersion: "33711" uid: 85ad8214-5423-4ac0-9d31-013348171ec2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-43223-predictor-5968f7b98d namespace: kserve-ci-e2e-test resourceVersion: "25999" uid: c9bdd6a7-8188-4b0f-8951-21eaecb3e020 kind: Event lastTimestamp: "2026-07-01T19:09:08Z" message: 'Created pod: error-404-isvc-43223-predictor-5968f7b98d-s44nj' metadata: creationTimestamp: "2026-07-01T19:09:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:09:08Z" name: error-404-isvc-43223-predictor-5968f7b98d.18be40120be296bc namespace: kserve-ci-e2e-test resourceVersion: "26010" uid: e74a4592-dc58-4a45-918f-babf19713889 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-43223-predictor namespace: kserve-ci-e2e-test resourceVersion: "25995" uid: 2da4bb14-d60e-4625-af7f-85533139d464 kind: Event lastTimestamp: "2026-07-01T19:09:08Z" message: Scaled up replica set error-404-isvc-43223-predictor-5968f7b98d from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:09:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:09:08Z" name: error-404-isvc-43223-predictor.18be40120a19444a namespace: kserve-ci-e2e-test resourceVersion: "26000" uid: 9a046f38-0712-450c-b28c-dc4c157ffe88 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:09:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-43223-predictor namespace: kserve-ci-e2e-test resourceVersion: "25998" uid: 657d757b-0dad-493b-977a-83d15be314ae kind: Event lastTimestamp: "2026-07-01T19:09:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:09:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:09:38Z" name: error-404-isvc-43223-predictor.18be401588b640b3 namespace: kserve-ci-e2e-test resourceVersion: "26462" uid: b7ffa639-7478-49ca-8981-8fba1d91026f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:09:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-43223-predictor namespace: kserve-ci-e2e-test resourceVersion: "25998" uid: 657d757b-0dad-493b-977a-83d15be314ae kind: Event lastTimestamp: "2026-07-01T19:09:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:09:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:09:38Z" name: error-404-isvc-43223-predictor.18be401589313db0 namespace: kserve-ci-e2e-test resourceVersion: "26463" uid: 1a3e248c-194c-4982-8e60-e1d678e05146 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:09:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-43223-predictor namespace: kserve-ci-e2e-test resourceVersion: "26332" uid: 657d757b-0dad-493b-977a-83d15be314ae kind: Event lastTimestamp: "2026-07-01T19:10:08Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:09:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:10:08Z" name: error-404-isvc-43223-predictor.18be401c8766af07 namespace: kserve-ci-e2e-test resourceVersion: "26735" uid: cfd44e88-9cd0-4caf-9402-8e43fe018049 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:09:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-43223-predictor namespace: kserve-ci-e2e-test resourceVersion: "26332" uid: 657d757b-0dad-493b-977a-83d15be314ae kind: Event lastTimestamp: "2026-07-01T19:10:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:09:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:10:08Z" name: error-404-isvc-43223-predictor.18be401c87db5bdc namespace: kserve-ci-e2e-test resourceVersion: "26736" uid: 89c77796-92af-4b79-ae6c-9b63ebd7a63b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:09:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-43223 namespace: kserve-ci-e2e-test resourceVersion: "25971" uid: 8cf6c540-d570-4ea5-8559-27bcf8103dca kind: Event lastTimestamp: "2026-07-01T19:09:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:09:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:09:57Z" name: error-404-isvc-43223.18be40120a2977b2 namespace: kserve-ci-e2e-test resourceVersion: "26646" uid: 66e18fd9-03a4-4292-95f8-ad3ac275b1ee reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-43223 namespace: kserve-ci-e2e-test resourceVersion: "25971" uid: 8cf6c540-d570-4ea5-8559-27bcf8103dca kind: Event lastTimestamp: "2026-07-01T19:09:08Z" message: 'Failed to update status for InferenceService "error-404-isvc-43223": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-43223": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:09:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:09:08Z" name: error-404-isvc-43223.18be40120acad024 namespace: kserve-ci-e2e-test resourceVersion: "26004" uid: 32193ea0-adf5-43f5-a437-cb07932c3616 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-43223 namespace: kserve-ci-e2e-test resourceVersion: "25971" uid: 8cf6c540-d570-4ea5-8559-27bcf8103dca kind: Event lastTimestamp: "2026-07-01T19:09:08Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-43223": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:09:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:09:08Z" name: error-404-isvc-43223.18be40120acb3003 namespace: kserve-ci-e2e-test resourceVersion: "26008" uid: 34d28d3f-ce98-4f2e-ad15-2c2bac6640b3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-43223 namespace: kserve-ci-e2e-test resourceVersion: "26638" uid: 8cf6c540-d570-4ea5-8559-27bcf8103dca kind: Event lastTimestamp: "2026-07-01T19:09:57Z" message: InferenceService [error-404-isvc-43223] is Ready metadata: creationTimestamp: "2026-07-01T19:09:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:09:57Z" name: error-404-isvc-43223.18be401d48e4f1ae namespace: kserve-ci-e2e-test resourceVersion: "26639" uid: 3bb17317-9c0e-4b8d-8d94-cc3be8cc35b1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:09:09.269523Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4f643-predictor-7879dc7bcd-279lm namespace: kserve-ci-e2e-test resourceVersion: "26124" uid: 997026bf-99b4-4bbf-8ac8-415238fa80af kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4f643-predictor-7879dc7bcd-279lm to ip-10-0-129-245.ec2.internal metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:09:09Z" name: error-404-isvc-4f643-predictor-7879dc7bcd-279lm.18be40122b8ccb09 namespace: kserve-ci-e2e-test resourceVersion: "26128" uid: d7cdc3ea-ab31-40a8-9550-ff2a69790de7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4f643-predictor-7879dc7bcd-279lm namespace: kserve-ci-e2e-test resourceVersion: "26126" uid: 997026bf-99b4-4bbf-8ac8-415238fa80af kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-4f643-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:09Z" name: error-404-isvc-4f643-predictor-7879dc7bcd-279lm.18be401234e7da73 namespace: kserve-ci-e2e-test resourceVersion: "26142" uid: 441ed15a-7dbb-49dc-b904-77133b7b4e02 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:10Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4f643-predictor-7879dc7bcd-279lm namespace: kserve-ci-e2e-test resourceVersion: "26133" uid: 997026bf-99b4-4bbf-8ac8-415238fa80af kind: Event lastTimestamp: "2026-07-01T19:09:10Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:09:10Z" name: error-404-isvc-4f643-predictor-7879dc7bcd-279lm.18be40126a1883d9 namespace: kserve-ci-e2e-test resourceVersion: "26182" uid: 2ce4c92e-6db0-4802-8e90-18ae9d9011ca reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4f643-predictor-7879dc7bcd-279lm namespace: kserve-ci-e2e-test resourceVersion: "26126" uid: 997026bf-99b4-4bbf-8ac8-415238fa80af kind: Event lastTimestamp: "2026-07-01T19:09:10Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:10Z" name: error-404-isvc-4f643-predictor-7879dc7bcd-279lm.18be40126bd90918 namespace: kserve-ci-e2e-test resourceVersion: "26184" uid: 0f423d20-654f-4d46-bf49-f3847411846f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4f643-predictor-7879dc7bcd-279lm namespace: kserve-ci-e2e-test resourceVersion: "26126" uid: 997026bf-99b4-4bbf-8ac8-415238fa80af kind: Event lastTimestamp: "2026-07-01T19:09:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:10Z" name: error-404-isvc-4f643-predictor-7879dc7bcd-279lm.18be401271acd03a namespace: kserve-ci-e2e-test resourceVersion: "26188" uid: 89c2f8e1-0752-4b9c-ac3d-c43dac20344e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4f643-predictor-7879dc7bcd-279lm namespace: kserve-ci-e2e-test resourceVersion: "26126" uid: 997026bf-99b4-4bbf-8ac8-415238fa80af kind: Event lastTimestamp: "2026-07-01T19:09:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:10Z" name: error-404-isvc-4f643-predictor-7879dc7bcd-279lm.18be401272290330 namespace: kserve-ci-e2e-test resourceVersion: "26190" uid: 6b862794-6290-4449-85f9-6e97b1207ab1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4f643-predictor-7879dc7bcd-279lm namespace: kserve-ci-e2e-test resourceVersion: "26126" uid: 997026bf-99b4-4bbf-8ac8-415238fa80af kind: Event lastTimestamp: "2026-07-01T19:09:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:10Z" name: error-404-isvc-4f643-predictor-7879dc7bcd-279lm.18be40127233fe5c namespace: kserve-ci-e2e-test resourceVersion: "26191" uid: a2f1ffaa-1fdc-42ed-8b48-e6673b45e095 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4f643-predictor-7879dc7bcd-279lm namespace: kserve-ci-e2e-test resourceVersion: "26126" uid: 997026bf-99b4-4bbf-8ac8-415238fa80af kind: Event lastTimestamp: "2026-07-01T19:09:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:10Z" name: error-404-isvc-4f643-predictor-7879dc7bcd-279lm.18be401274cc7e98 namespace: kserve-ci-e2e-test resourceVersion: "26192" uid: 74db019a-079b-4e18-affc-306508aa276d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4f643-predictor-7879dc7bcd-279lm namespace: kserve-ci-e2e-test resourceVersion: "26126" uid: 997026bf-99b4-4bbf-8ac8-415238fa80af kind: Event lastTimestamp: "2026-07-01T19:09:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:10Z" name: error-404-isvc-4f643-predictor-7879dc7bcd-279lm.18be40127539f0cd namespace: kserve-ci-e2e-test resourceVersion: "26193" uid: 1401489a-e729-46c4-8e80-5b45dbf79418 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4f643-predictor-7879dc7bcd-279lm namespace: kserve-ci-e2e-test resourceVersion: "26126" uid: 997026bf-99b4-4bbf-8ac8-415238fa80af kind: Event lastTimestamp: "2026-07-01T19:18:26Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:18:26Z" name: error-404-isvc-4f643-predictor-7879dc7bcd-279lm.18be40128c911cf7 namespace: kserve-ci-e2e-test resourceVersion: "30315" uid: 6e693d1a-f76f-49e6-8578-993aa68ca1b0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:18:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4f643-predictor-7879dc7bcd-279lm namespace: kserve-ci-e2e-test resourceVersion: "26126" uid: 997026bf-99b4-4bbf-8ac8-415238fa80af kind: Event lastTimestamp: "2026-07-01T19:18:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:18:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:18:24Z" name: error-404-isvc-4f643-predictor-7879dc7bcd-279lm.18be40935a65ab96 namespace: kserve-ci-e2e-test resourceVersion: "30289" uid: 580d2067-8682-4ebe-aaec-3ef7baf20789 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:18:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4f643-predictor-7879dc7bcd-279lm namespace: kserve-ci-e2e-test resourceVersion: "26126" uid: 997026bf-99b4-4bbf-8ac8-415238fa80af kind: Event lastTimestamp: "2026-07-01T19:18:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:18:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:18:24Z" name: error-404-isvc-4f643-predictor-7879dc7bcd-279lm.18be40935a662869 namespace: kserve-ci-e2e-test resourceVersion: "30290" uid: 7b3d223b-d0a3-4a8b-9285-e90ff70047d8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:18:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4f643-predictor-7879dc7bcd-279lm namespace: kserve-ci-e2e-test resourceVersion: "26126" uid: 997026bf-99b4-4bbf-8ac8-415238fa80af kind: Event lastTimestamp: "2026-07-01T19:18:26Z" message: 'Readiness probe failed: Get "https://10.134.0.40:8643/healthz": dial tcp 10.134.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:18:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:18:26Z" name: error-404-isvc-4f643-predictor-7879dc7bcd-279lm.18be40940112c56a namespace: kserve-ci-e2e-test resourceVersion: "30314" uid: 9ea138c9-dd81-4fbc-b09e-b5c56dff81f3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4f643-predictor-7879dc7bcd namespace: kserve-ci-e2e-test resourceVersion: "26117" uid: d842393b-6020-41a6-b231-9334acaa1fc6 kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: 'Created pod: error-404-isvc-4f643-predictor-7879dc7bcd-279lm' metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:09:09Z" name: error-404-isvc-4f643-predictor-7879dc7bcd.18be40122afb00e1 namespace: kserve-ci-e2e-test resourceVersion: "26125" uid: 8f4746bd-5fd5-439d-ae7f-35860dea0244 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4f643-predictor namespace: kserve-ci-e2e-test resourceVersion: "26115" uid: 76ad360f-e3db-404e-832a-8cc88b04cdc8 kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: Scaled up replica set error-404-isvc-4f643-predictor-7879dc7bcd from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:09:09Z" name: error-404-isvc-4f643-predictor.18be401229550df4 namespace: kserve-ci-e2e-test resourceVersion: "26118" uid: ada284f2-d298-4ffd-87f6-78b1b303e6a8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:09:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4f643-predictor namespace: kserve-ci-e2e-test resourceVersion: "26116" uid: 47b35abe-d9be-4890-9aed-c1c8ac256c6d kind: Event lastTimestamp: "2026-07-01T19:09:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:09:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:09:39Z" name: error-404-isvc-4f643-predictor.18be4015a7c9ca14 namespace: kserve-ci-e2e-test resourceVersion: "26469" uid: bfb4ad61-250d-4774-b75c-84d438ada492 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:09:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4f643-predictor namespace: kserve-ci-e2e-test resourceVersion: "26116" uid: 47b35abe-d9be-4890-9aed-c1c8ac256c6d kind: Event lastTimestamp: "2026-07-01T19:09:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:09:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:09:39Z" name: error-404-isvc-4f643-predictor.18be4015a843f5f4 namespace: kserve-ci-e2e-test resourceVersion: "26470" uid: 54b917ac-0d54-451a-8068-10ea8b7c7866 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:09:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4f643-predictor namespace: kserve-ci-e2e-test resourceVersion: "26343" uid: 47b35abe-d9be-4890-9aed-c1c8ac256c6d kind: Event lastTimestamp: "2026-07-01T19:10:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:09:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:10:24Z" name: error-404-isvc-4f643-predictor.18be401ca5fccf20 namespace: kserve-ci-e2e-test resourceVersion: "26938" uid: 2c45c9dd-05fe-4c94-b1bd-2ac8e3468c1b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:09:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4f643-predictor namespace: kserve-ci-e2e-test resourceVersion: "26343" uid: 47b35abe-d9be-4890-9aed-c1c8ac256c6d kind: Event lastTimestamp: "2026-07-01T19:10:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:09:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:10:24Z" name: error-404-isvc-4f643-predictor.18be401ca66d1bf8 namespace: kserve-ci-e2e-test resourceVersion: "26939" uid: 0c895100-5b04-4fc7-bf40-4cd202984788 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4f643 namespace: kserve-ci-e2e-test resourceVersion: "26096" uid: dd882da3-b6e2-447b-9b9a-379b82b7c32e kind: Event lastTimestamp: "2026-07-01T19:09:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:09:57Z" name: error-404-isvc-4f643.18be4012295a47c5 namespace: kserve-ci-e2e-test resourceVersion: "26647" uid: 56b4fb79-2b15-4d2d-8b70-eb6d9272a9d5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4f643 namespace: kserve-ci-e2e-test resourceVersion: "26096" uid: dd882da3-b6e2-447b-9b9a-379b82b7c32e kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: 'Failed to update status for InferenceService "error-404-isvc-4f643": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4f643": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:09:09Z" name: error-404-isvc-4f643.18be401229d39a7e namespace: kserve-ci-e2e-test resourceVersion: "26121" uid: eb4e66fd-b0ae-4920-9edc-843f600a193b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4f643 namespace: kserve-ci-e2e-test resourceVersion: "26096" uid: dd882da3-b6e2-447b-9b9a-379b82b7c32e kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4f643": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:09:09Z" name: error-404-isvc-4f643.18be401229d3d63f namespace: kserve-ci-e2e-test resourceVersion: "26122" uid: 4776892e-1e43-4236-861c-7af19e474317 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4f643 namespace: kserve-ci-e2e-test resourceVersion: "26641" uid: dd882da3-b6e2-447b-9b9a-379b82b7c32e kind: Event lastTimestamp: "2026-07-01T19:09:57Z" message: InferenceService [error-404-isvc-4f643] is Ready metadata: creationTimestamp: "2026-07-01T19:09:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:09:57Z" name: error-404-isvc-4f643.18be401d4c0a2f4c namespace: kserve-ci-e2e-test resourceVersion: "26642" uid: c7535b70-125b-4b40-8425-dbb7f9b6f11c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:58:29.582965Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-550f3-predictor-787558b8bb-22gl7 namespace: kserve-ci-e2e-test resourceVersion: "20278" uid: 247c587d-b6d5-4fff-a212-771964aecffe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-550f3-predictor-787558b8bb-22gl7 to ip-10-0-129-245.ec2.internal metadata: creationTimestamp: "2026-07-01T18:58:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:58:29Z" name: error-404-isvc-550f3-predictor-787558b8bb-22gl7.18be3f7d3b429ec5 namespace: kserve-ci-e2e-test resourceVersion: "20282" uid: 708b3e37-8925-4e5f-a348-4e96fafda754 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:30Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-550f3-predictor-787558b8bb-22gl7 namespace: kserve-ci-e2e-test resourceVersion: "20286" uid: 247c587d-b6d5-4fff-a212-771964aecffe kind: Event lastTimestamp: "2026-07-01T18:58:30Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:58:30Z" name: error-404-isvc-550f3-predictor-787558b8bb-22gl7.18be3f7d5587174e namespace: kserve-ci-e2e-test resourceVersion: "20306" uid: f65521ae-740f-4509-816d-afda400e7ec4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-550f3-predictor-787558b8bb-22gl7 namespace: kserve-ci-e2e-test resourceVersion: "20279" uid: 247c587d-b6d5-4fff-a212-771964aecffe kind: Event lastTimestamp: "2026-07-01T18:58:30Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T18:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:30Z" name: error-404-isvc-550f3-predictor-787558b8bb-22gl7.18be3f7d57390ea9 namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: 18739279-0140-4c07-b417-1fa0af60e41f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-550f3-predictor-787558b8bb-22gl7 namespace: kserve-ci-e2e-test resourceVersion: "20279" uid: 247c587d-b6d5-4fff-a212-771964aecffe kind: Event lastTimestamp: "2026-07-01T18:58:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:30Z" name: error-404-isvc-550f3-predictor-787558b8bb-22gl7.18be3f7d5a413a6e namespace: kserve-ci-e2e-test resourceVersion: "20311" uid: 1d1ed05b-05a7-44ee-a33a-014a67047a39 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-550f3-predictor-787558b8bb-22gl7 namespace: kserve-ci-e2e-test resourceVersion: "20279" uid: 247c587d-b6d5-4fff-a212-771964aecffe kind: Event lastTimestamp: "2026-07-01T18:58:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:30Z" name: error-404-isvc-550f3-predictor-787558b8bb-22gl7.18be3f7d5aabe19b namespace: kserve-ci-e2e-test resourceVersion: "20312" uid: 37cfeeb6-ea03-497f-84e0-6e8a487bd178 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-550f3-predictor-787558b8bb-22gl7 namespace: kserve-ci-e2e-test resourceVersion: "20279" uid: 247c587d-b6d5-4fff-a212-771964aecffe kind: Event lastTimestamp: "2026-07-01T18:58:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:30Z" name: error-404-isvc-550f3-predictor-787558b8bb-22gl7.18be3f7d5ab4e4e7 namespace: kserve-ci-e2e-test resourceVersion: "20313" uid: b023701f-8cdc-44f0-ba98-61ee2579e6f3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-550f3-predictor-787558b8bb-22gl7 namespace: kserve-ci-e2e-test resourceVersion: "20279" uid: 247c587d-b6d5-4fff-a212-771964aecffe kind: Event lastTimestamp: "2026-07-01T18:58:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:30Z" name: error-404-isvc-550f3-predictor-787558b8bb-22gl7.18be3f7d5decb102 namespace: kserve-ci-e2e-test resourceVersion: "20315" uid: f18f0d2b-551d-42fe-ab91-21e775c60484 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-550f3-predictor-787558b8bb-22gl7 namespace: kserve-ci-e2e-test resourceVersion: "20279" uid: 247c587d-b6d5-4fff-a212-771964aecffe kind: Event lastTimestamp: "2026-07-01T18:58:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:30Z" name: error-404-isvc-550f3-predictor-787558b8bb-22gl7.18be3f7d5e5c710e namespace: kserve-ci-e2e-test resourceVersion: "20316" uid: 22b64a1a-08cf-4e1c-9af3-597dd30eb4c5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:58:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-550f3-predictor-787558b8bb-22gl7 namespace: kserve-ci-e2e-test resourceVersion: "20279" uid: 247c587d-b6d5-4fff-a212-771964aecffe kind: Event lastTimestamp: "2026-07-01T18:59:07Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:58:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:07Z" name: error-404-isvc-550f3-predictor-787558b8bb-22gl7.18be3f7da5c952e8 namespace: kserve-ci-e2e-test resourceVersion: "20835" uid: f7f0b083-b750-4845-9bc2-46f0e9674e93 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-550f3-predictor-787558b8bb-22gl7 namespace: kserve-ci-e2e-test resourceVersion: "20279" uid: 247c587d-b6d5-4fff-a212-771964aecffe kind: Event lastTimestamp: "2026-07-01T18:59:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:53Z" name: error-404-isvc-550f3-predictor-787558b8bb-22gl7.18be3f90d34d60ed namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: 51a9375e-ad81-452c-9c83-400951c6fe7f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-550f3-predictor-787558b8bb-22gl7 namespace: kserve-ci-e2e-test resourceVersion: "20279" uid: 247c587d-b6d5-4fff-a212-771964aecffe kind: Event lastTimestamp: "2026-07-01T18:59:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:53Z" name: error-404-isvc-550f3-predictor-787558b8bb-22gl7.18be3f90d34dc68d namespace: kserve-ci-e2e-test resourceVersion: "21370" uid: 3ba28c06-4e7b-4313-8005-d7ca867cc22d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-550f3-predictor-787558b8bb namespace: kserve-ci-e2e-test resourceVersion: "20271" uid: ac2f094b-0131-4e29-8347-b95fede7e01f kind: Event lastTimestamp: "2026-07-01T18:58:29Z" message: 'Created pod: error-404-isvc-550f3-predictor-787558b8bb-22gl7' metadata: creationTimestamp: "2026-07-01T18:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:58:29Z" name: error-404-isvc-550f3-predictor-787558b8bb.18be3f7d3ad0aa06 namespace: kserve-ci-e2e-test resourceVersion: "20280" uid: 36ebab7a-c71c-4b28-88d1-ef3632d346f4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-550f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 7292a08d-3056-422d-a74b-48a815efe3dd kind: Event lastTimestamp: "2026-07-01T18:58:29Z" message: Scaled up replica set error-404-isvc-550f3-predictor-787558b8bb from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:58:29Z" name: error-404-isvc-550f3-predictor.18be3f7d391f5eff namespace: kserve-ci-e2e-test resourceVersion: "20272" uid: 56059d78-934f-4093-9632-df170d2ecf44 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:58:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-550f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20270" uid: e334a508-47c4-4833-8d33-27e68c088355 kind: Event lastTimestamp: "2026-07-01T18:58:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:58:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:58:59Z" name: error-404-isvc-550f3-predictor.18be3f80b7cbd43b namespace: kserve-ci-e2e-test resourceVersion: "20769" uid: 8567305d-0f1f-4a0a-8b5e-5b496ee282f3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:58:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-550f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20270" uid: e334a508-47c4-4833-8d33-27e68c088355 kind: Event lastTimestamp: "2026-07-01T18:58:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:58:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:58:59Z" name: error-404-isvc-550f3-predictor.18be3f80b83ffb94 namespace: kserve-ci-e2e-test resourceVersion: "20770" uid: f5a1b486-8735-45a8-8999-c74d60fd122d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:59:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-550f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20609" uid: e334a508-47c4-4833-8d33-27e68c088355 kind: Event lastTimestamp: "2026-07-01T18:59:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:59:44Z" name: error-404-isvc-550f3-predictor.18be3f87b5eabe2e namespace: kserve-ci-e2e-test resourceVersion: "21222" uid: 1e54598d-da24-4835-b5b5-bd8304a30047 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:59:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-550f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20609" uid: e334a508-47c4-4833-8d33-27e68c088355 kind: Event lastTimestamp: "2026-07-01T18:59:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:59:44Z" name: error-404-isvc-550f3-predictor.18be3f87b63f4548 namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: e620c1a3-999e-41a7-9987-d326acdd8658 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:58:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-550f3 namespace: kserve-ci-e2e-test resourceVersion: "20256" uid: 872fd3c6-e695-4cb5-8cbf-4fbe3b6ab318 kind: Event lastTimestamp: "2026-07-01T18:59:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:59:17Z" name: error-404-isvc-550f3.18be3f7d3922ef5a namespace: kserve-ci-e2e-test resourceVersion: "20925" uid: 5e300bfc-4582-4887-8515-53e75088b6e6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-550f3 namespace: kserve-ci-e2e-test resourceVersion: "20256" uid: 872fd3c6-e695-4cb5-8cbf-4fbe3b6ab318 kind: Event lastTimestamp: "2026-07-01T18:58:29Z" message: 'Failed to update status for InferenceService "error-404-isvc-550f3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-550f3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:58:29Z" name: error-404-isvc-550f3.18be3f7d39b58d1a namespace: kserve-ci-e2e-test resourceVersion: "20275" uid: 3e1b7ab4-204b-4c61-b569-f8db7959817c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-550f3 namespace: kserve-ci-e2e-test resourceVersion: "20256" uid: 872fd3c6-e695-4cb5-8cbf-4fbe3b6ab318 kind: Event lastTimestamp: "2026-07-01T18:58:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-550f3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:58:29Z" name: error-404-isvc-550f3.18be3f7d39b5eb3e namespace: kserve-ci-e2e-test resourceVersion: "20276" uid: 99b901f3-bcdf-4275-8861-ed141bd4c057 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-550f3 namespace: kserve-ci-e2e-test resourceVersion: "20923" uid: 872fd3c6-e695-4cb5-8cbf-4fbe3b6ab318 kind: Event lastTimestamp: "2026-07-01T18:59:17Z" message: InferenceService [error-404-isvc-550f3] is Ready metadata: creationTimestamp: "2026-07-01T18:59:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:59:17Z" name: error-404-isvc-550f3.18be3f88622e0cd3 namespace: kserve-ci-e2e-test resourceVersion: "20924" uid: 0fa06f3d-8cae-40ab-a46b-5fb720e4507f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:48:18.892804Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9857d-predictor-74fddbbbc8-ftl8h namespace: kserve-ci-e2e-test resourceVersion: "15264" uid: 0d99116f-f1da-4605-b102-292b79631e8f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-9857d-predictor-74fddbbbc8-ftl8h to ip-10-0-129-245.ec2.internal metadata: creationTimestamp: "2026-07-01T18:48:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:48:18Z" name: error-404-isvc-9857d-predictor-74fddbbbc8-ftl8h.18be3eef0b4a3627 namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: 594d48c7-db1a-47d9-964a-8911638ff92c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:19Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9857d-predictor-74fddbbbc8-ftl8h namespace: kserve-ci-e2e-test resourceVersion: "15280" uid: 0d99116f-f1da-4605-b102-292b79631e8f kind: Event lastTimestamp: "2026-07-01T18:48:19Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:48:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:48:19Z" name: error-404-isvc-9857d-predictor-74fddbbbc8-ftl8h.18be3eef24d760c7 namespace: kserve-ci-e2e-test resourceVersion: "15303" uid: b335627f-3376-4e99-892b-f065d3886eb0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9857d-predictor-74fddbbbc8-ftl8h namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: 0d99116f-f1da-4605-b102-292b79631e8f kind: Event lastTimestamp: "2026-07-01T18:48:19Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T18:48:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:19Z" name: error-404-isvc-9857d-predictor-74fddbbbc8-ftl8h.18be3eef267ec3aa namespace: kserve-ci-e2e-test resourceVersion: "15307" uid: 53c9a422-f550-48f3-8ee5-d6acaa81e91e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9857d-predictor-74fddbbbc8-ftl8h namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: 0d99116f-f1da-4605-b102-292b79631e8f kind: Event lastTimestamp: "2026-07-01T18:48:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:48:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:19Z" name: error-404-isvc-9857d-predictor-74fddbbbc8-ftl8h.18be3eef29c80908 namespace: kserve-ci-e2e-test resourceVersion: "15308" uid: 26523887-f5c0-408a-9d66-0443cde0f94e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9857d-predictor-74fddbbbc8-ftl8h namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: 0d99116f-f1da-4605-b102-292b79631e8f kind: Event lastTimestamp: "2026-07-01T18:48:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:48:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:19Z" name: error-404-isvc-9857d-predictor-74fddbbbc8-ftl8h.18be3eef2a5dbcc2 namespace: kserve-ci-e2e-test resourceVersion: "15309" uid: 097c0a05-99ef-4565-8a59-5dbaac187c3c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9857d-predictor-74fddbbbc8-ftl8h namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: 0d99116f-f1da-4605-b102-292b79631e8f kind: Event lastTimestamp: "2026-07-01T18:48:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:48:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:19Z" name: error-404-isvc-9857d-predictor-74fddbbbc8-ftl8h.18be3eef2a68763f namespace: kserve-ci-e2e-test resourceVersion: "15310" uid: 83736633-96f1-4ad9-a811-248e6ebf49fc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9857d-predictor-74fddbbbc8-ftl8h namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: 0d99116f-f1da-4605-b102-292b79631e8f kind: Event lastTimestamp: "2026-07-01T18:48:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:48:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:19Z" name: error-404-isvc-9857d-predictor-74fddbbbc8-ftl8h.18be3eef2d5da265 namespace: kserve-ci-e2e-test resourceVersion: "15311" uid: 0725b1b5-8e1e-4386-bf56-12aecc9cd006 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9857d-predictor-74fddbbbc8-ftl8h namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: 0d99116f-f1da-4605-b102-292b79631e8f kind: Event lastTimestamp: "2026-07-01T18:48:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:48:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:19Z" name: error-404-isvc-9857d-predictor-74fddbbbc8-ftl8h.18be3eef2dd74896 namespace: kserve-ci-e2e-test resourceVersion: "15312" uid: 6ccdf644-71b2-441e-978b-56edc83c7358 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T18:48:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9857d-predictor-74fddbbbc8-ftl8h namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: 0d99116f-f1da-4605-b102-292b79631e8f kind: Event lastTimestamp: "2026-07-01T18:57:36Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:48:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:57:36Z" name: error-404-isvc-9857d-predictor-74fddbbbc8-ftl8h.18be3eef5085df42 namespace: kserve-ci-e2e-test resourceVersion: "19759" uid: 52ce2503-091b-42b6-96a9-c1af220c10e7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9857d-predictor-74fddbbbc8-ftl8h namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: 0d99116f-f1da-4605-b102-292b79631e8f kind: Event lastTimestamp: "2026-07-01T18:57:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:57:33Z" name: error-404-isvc-9857d-predictor-74fddbbbc8-ftl8h.18be3f703c21406b namespace: kserve-ci-e2e-test resourceVersion: "19702" uid: 0f074110-6c6b-4fc7-99f9-98cd267204a7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9857d-predictor-74fddbbbc8-ftl8h namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: 0d99116f-f1da-4605-b102-292b79631e8f kind: Event lastTimestamp: "2026-07-01T18:57:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:57:33Z" name: error-404-isvc-9857d-predictor-74fddbbbc8-ftl8h.18be3f703c2244eb namespace: kserve-ci-e2e-test resourceVersion: "19703" uid: 45cf8b00-ec9d-4ac4-9866-2ff2de1cd50f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9857d-predictor-74fddbbbc8-ftl8h namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: 0d99116f-f1da-4605-b102-292b79631e8f kind: Event lastTimestamp: "2026-07-01T18:57:36Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:57:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:57:36Z" name: error-404-isvc-9857d-predictor-74fddbbbc8-ftl8h.18be3f70c4fa6316 namespace: kserve-ci-e2e-test resourceVersion: "19757" uid: beeb7078-016f-4cc9-822e-9841598ac73b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-9857d-predictor-74fddbbbc8 namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: 78914bd1-8dfc-454f-8380-444618ecdcfa kind: Event lastTimestamp: "2026-07-01T18:48:18Z" message: 'Created pod: error-404-isvc-9857d-predictor-74fddbbbc8-ftl8h' metadata: creationTimestamp: "2026-07-01T18:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:48:18Z" name: error-404-isvc-9857d-predictor-74fddbbbc8.18be3eef0a82f0f3 namespace: kserve-ci-e2e-test resourceVersion: "15265" uid: 0de700ef-54ce-4022-81fa-ca6a6a0ec1f4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-9857d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15252" uid: 75b1d767-8f15-4c67-b186-f8af3acc1751 kind: Event lastTimestamp: "2026-07-01T18:48:18Z" message: Scaled up replica set error-404-isvc-9857d-predictor-74fddbbbc8 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:48:18Z" name: error-404-isvc-9857d-predictor.18be3eef08a7e84e namespace: kserve-ci-e2e-test resourceVersion: "15257" uid: 760424dc-7989-4c5a-b3cd-8e375ff7f0e9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:48:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9857d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15254" uid: d7e8918e-4244-492d-83ac-982f30dd1ea1 kind: Event lastTimestamp: "2026-07-01T18:49:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:48:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:49:03Z" name: error-404-isvc-9857d-predictor.18be3ef2875e0fd6 namespace: kserve-ci-e2e-test resourceVersion: "15723" uid: fc74eae1-18e3-4ae3-a8f9-3cbaf812676e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:48:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9857d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15254" uid: d7e8918e-4244-492d-83ac-982f30dd1ea1 kind: Event lastTimestamp: "2026-07-01T18:49:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:48:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:49:03Z" name: error-404-isvc-9857d-predictor.18be3ef287cb49cc namespace: kserve-ci-e2e-test resourceVersion: "15724" uid: 641788aa-d5da-401b-98c5-495d837c798c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:48:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9857d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15466" uid: d7e8918e-4244-492d-83ac-982f30dd1ea1 kind: Event lastTimestamp: "2026-07-01T18:49:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:49:33Z" name: error-404-isvc-9857d-predictor.18be3ef606a57323 namespace: kserve-ci-e2e-test resourceVersion: "16227" uid: 639b62ac-65b5-45f9-a65f-e8c9808fa142 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:48:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9857d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15466" uid: d7e8918e-4244-492d-83ac-982f30dd1ea1 kind: Event lastTimestamp: "2026-07-01T18:49:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:49:33Z" name: error-404-isvc-9857d-predictor.18be3ef6072b11b4 namespace: kserve-ci-e2e-test resourceVersion: "16228" uid: c1a77ea3-aea2-49f8-837c-6a6a76b78c86 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:48:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9857d namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 541a7f78-5f2d-4723-8ca6-8f17f4948875 kind: Event lastTimestamp: "2026-07-01T18:49:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:49:06Z" name: error-404-isvc-9857d.18be3eef0902e692 namespace: kserve-ci-e2e-test resourceVersion: "15790" uid: eb801da4-b016-4974-9490-066c781b2ac1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9857d namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 541a7f78-5f2d-4723-8ca6-8f17f4948875 kind: Event lastTimestamp: "2026-07-01T18:48:18Z" message: 'Failed to update status for InferenceService "error-404-isvc-9857d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9857d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:48:18Z" name: error-404-isvc-9857d.18be3eef097e5f39 namespace: kserve-ci-e2e-test resourceVersion: "15261" uid: 7c750b9a-9ea7-4934-88db-e11825298686 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9857d namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 541a7f78-5f2d-4723-8ca6-8f17f4948875 kind: Event lastTimestamp: "2026-07-01T18:48:18Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9857d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:48:18Z" name: error-404-isvc-9857d.18be3eef097eb01e namespace: kserve-ci-e2e-test resourceVersion: "15263" uid: 1e6179c0-25cd-4df9-b90c-42d977f06a0e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9857d namespace: kserve-ci-e2e-test resourceVersion: "15788" uid: 541a7f78-5f2d-4723-8ca6-8f17f4948875 kind: Event lastTimestamp: "2026-07-01T18:49:06Z" message: InferenceService [error-404-isvc-9857d] is Ready metadata: creationTimestamp: "2026-07-01T18:49:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:49:06Z" name: error-404-isvc-9857d.18be3efa0c9ce18d namespace: kserve-ci-e2e-test resourceVersion: "15789" uid: e3dcbd04-78bd-4593-b113-afbb7c2fcd31 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:58:44.004450Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-af621-predictor-6c7cdbfc8d-gsrwd namespace: kserve-ci-e2e-test resourceVersion: "20577" uid: dcffa820-58d6-4ab1-8975-e6adcaf9f0de kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-af621-predictor-6c7cdbfc8d-gsrwd to ip-10-0-129-245.ec2.internal metadata: creationTimestamp: "2026-07-01T18:58:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:58:44Z" name: error-404-isvc-af621-predictor-6c7cdbfc8d-gsrwd.18be3f8096d90100 namespace: kserve-ci-e2e-test resourceVersion: "20581" uid: 848ada9a-fd95-48b0-9793-e1813777ada5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:44Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-af621-predictor-6c7cdbfc8d-gsrwd namespace: kserve-ci-e2e-test resourceVersion: "20579" uid: dcffa820-58d6-4ab1-8975-e6adcaf9f0de kind: Event lastTimestamp: "2026-07-01T18:58:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-af621-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:58:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:44Z" name: error-404-isvc-af621-predictor-6c7cdbfc8d-gsrwd.18be3f80a1bc0b3d namespace: kserve-ci-e2e-test resourceVersion: "20596" uid: 70c33a5f-2672-4f7c-88fb-e3e23ade7617 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:45Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-af621-predictor-6c7cdbfc8d-gsrwd namespace: kserve-ci-e2e-test resourceVersion: "20587" uid: dcffa820-58d6-4ab1-8975-e6adcaf9f0de kind: Event lastTimestamp: "2026-07-01T18:58:45Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:58:45Z" name: error-404-isvc-af621-predictor-6c7cdbfc8d-gsrwd.18be3f80d5990a50 namespace: kserve-ci-e2e-test resourceVersion: "20620" uid: 25888ff5-9e43-488a-957e-80d3fd18fa67 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-af621-predictor-6c7cdbfc8d-gsrwd namespace: kserve-ci-e2e-test resourceVersion: "20579" uid: dcffa820-58d6-4ab1-8975-e6adcaf9f0de kind: Event lastTimestamp: "2026-07-01T18:58:45Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T18:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:45Z" name: error-404-isvc-af621-predictor-6c7cdbfc8d-gsrwd.18be3f80d7048873 namespace: kserve-ci-e2e-test resourceVersion: "20622" uid: 89f8e4d7-12ed-4193-a175-9909c9a904b7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-af621-predictor-6c7cdbfc8d-gsrwd namespace: kserve-ci-e2e-test resourceVersion: "20579" uid: dcffa820-58d6-4ab1-8975-e6adcaf9f0de kind: Event lastTimestamp: "2026-07-01T18:58:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:45Z" name: error-404-isvc-af621-predictor-6c7cdbfc8d-gsrwd.18be3f80d9851082 namespace: kserve-ci-e2e-test resourceVersion: "20625" uid: 5ac6eb79-8e62-411a-8b77-d29380ac8ec8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-af621-predictor-6c7cdbfc8d-gsrwd namespace: kserve-ci-e2e-test resourceVersion: "20579" uid: dcffa820-58d6-4ab1-8975-e6adcaf9f0de kind: Event lastTimestamp: "2026-07-01T18:58:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:45Z" name: error-404-isvc-af621-predictor-6c7cdbfc8d-gsrwd.18be3f80d9f57807 namespace: kserve-ci-e2e-test resourceVersion: "20626" uid: 86d8efc5-2f9d-4c82-86c1-45b196dc99e3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-af621-predictor-6c7cdbfc8d-gsrwd namespace: kserve-ci-e2e-test resourceVersion: "20579" uid: dcffa820-58d6-4ab1-8975-e6adcaf9f0de kind: Event lastTimestamp: "2026-07-01T18:58:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:45Z" name: error-404-isvc-af621-predictor-6c7cdbfc8d-gsrwd.18be3f80da068c23 namespace: kserve-ci-e2e-test resourceVersion: "20627" uid: b2cdec5c-75e4-4bb6-8f41-e0ef6098a3a9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-af621-predictor-6c7cdbfc8d-gsrwd namespace: kserve-ci-e2e-test resourceVersion: "20579" uid: dcffa820-58d6-4ab1-8975-e6adcaf9f0de kind: Event lastTimestamp: "2026-07-01T18:58:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:45Z" name: error-404-isvc-af621-predictor-6c7cdbfc8d-gsrwd.18be3f80dde5ccc2 namespace: kserve-ci-e2e-test resourceVersion: "20628" uid: b888d480-f341-4229-8b1f-8b1719b9688c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-af621-predictor-6c7cdbfc8d-gsrwd namespace: kserve-ci-e2e-test resourceVersion: "20579" uid: dcffa820-58d6-4ab1-8975-e6adcaf9f0de kind: Event lastTimestamp: "2026-07-01T18:58:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:45Z" name: error-404-isvc-af621-predictor-6c7cdbfc8d-gsrwd.18be3f80de57e1ab namespace: kserve-ci-e2e-test resourceVersion: "20629" uid: 874484fb-237e-40ac-90ac-d6304fa51130 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:58:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-af621-predictor-6c7cdbfc8d-gsrwd namespace: kserve-ci-e2e-test resourceVersion: "20579" uid: dcffa820-58d6-4ab1-8975-e6adcaf9f0de kind: Event lastTimestamp: "2026-07-01T18:59:22Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:58:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:22Z" name: error-404-isvc-af621-predictor-6c7cdbfc8d-gsrwd.18be3f812894ca3d namespace: kserve-ci-e2e-test resourceVersion: "20957" uid: 472fddf6-8524-45ff-8c46-1fdf6483b7d9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:07:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-af621-predictor-6c7cdbfc8d-gsrwd namespace: kserve-ci-e2e-test resourceVersion: "20579" uid: dcffa820-58d6-4ab1-8975-e6adcaf9f0de kind: Event lastTimestamp: "2026-07-01T19:07:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:07:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:07:59Z" name: error-404-isvc-af621-predictor-6c7cdbfc8d-gsrwd.18be4001d361fe1c namespace: kserve-ci-e2e-test resourceVersion: "25010" uid: 77cb4b22-26cc-4536-85c9-5281259188e1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:07:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-af621-predictor-6c7cdbfc8d-gsrwd namespace: kserve-ci-e2e-test resourceVersion: "20579" uid: dcffa820-58d6-4ab1-8975-e6adcaf9f0de kind: Event lastTimestamp: "2026-07-01T19:07:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:07:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:07:59Z" name: error-404-isvc-af621-predictor-6c7cdbfc8d-gsrwd.18be4001d3629068 namespace: kserve-ci-e2e-test resourceVersion: "25012" uid: 44077f4d-e723-4468-9539-35274f7d83fe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-af621-predictor-6c7cdbfc8d namespace: kserve-ci-e2e-test resourceVersion: "20570" uid: f31641b2-4bd8-43cb-beac-c69f4363a6d9 kind: Event lastTimestamp: "2026-07-01T18:58:43Z" message: 'Created pod: error-404-isvc-af621-predictor-6c7cdbfc8d-gsrwd' metadata: creationTimestamp: "2026-07-01T18:58:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:58:43Z" name: error-404-isvc-af621-predictor-6c7cdbfc8d.18be3f80963f239a namespace: kserve-ci-e2e-test resourceVersion: "20578" uid: f5d4f72b-3f08-4dd0-bd34-3c1eab96dc25 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-af621-predictor namespace: kserve-ci-e2e-test resourceVersion: "20567" uid: ab963e9b-548c-4767-ab18-799f614f37d7 kind: Event lastTimestamp: "2026-07-01T18:58:43Z" message: Scaled up replica set error-404-isvc-af621-predictor-6c7cdbfc8d from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:58:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:58:43Z" name: error-404-isvc-af621-predictor.18be3f809478c136 namespace: kserve-ci-e2e-test resourceVersion: "20571" uid: f3d44f03-60e2-4f83-b3af-c48d9b801a1b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:58:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-af621-predictor namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: c019d9ae-16b3-401e-bd8a-8fea31e8ef8a kind: Event lastTimestamp: "2026-07-01T18:59:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:59:13Z" name: error-404-isvc-af621-predictor.18be3f841327e4c4 namespace: kserve-ci-e2e-test resourceVersion: "20882" uid: d2f43403-a784-41f7-8054-ce5133bda1f5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:58:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-af621-predictor namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: c019d9ae-16b3-401e-bd8a-8fea31e8ef8a kind: Event lastTimestamp: "2026-07-01T18:59:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:59:14Z" name: error-404-isvc-af621-predictor.18be3f8413a8d85e namespace: kserve-ci-e2e-test resourceVersion: "20883" uid: 9e9e58b9-7812-48f9-b74a-79f6e192e55b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:59:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-af621-predictor namespace: kserve-ci-e2e-test resourceVersion: "20759" uid: c019d9ae-16b3-401e-bd8a-8fea31e8ef8a kind: Event lastTimestamp: "2026-07-01T18:59:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:59:59Z" name: error-404-isvc-af621-predictor.18be3f8b117e0b3f namespace: kserve-ci-e2e-test resourceVersion: "21521" uid: 9574154e-4fc7-4d26-884b-826a40acd290 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:59:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-af621-predictor namespace: kserve-ci-e2e-test resourceVersion: "20759" uid: c019d9ae-16b3-401e-bd8a-8fea31e8ef8a kind: Event lastTimestamp: "2026-07-01T18:59:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:59:59Z" name: error-404-isvc-af621-predictor.18be3f8b11ee2f8a namespace: kserve-ci-e2e-test resourceVersion: "21522" uid: 67ceb620-dd24-414f-a4d2-e437e71bd84c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:58:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-af621 namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 50da2c58-ac14-40a4-ae72-360b9c011d67 kind: Event lastTimestamp: "2026-07-01T18:59:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:58:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:59:32Z" name: error-404-isvc-af621.18be3f80948125e6 namespace: kserve-ci-e2e-test resourceVersion: "21069" uid: 5b37323e-d256-4a3e-b016-d12e6062b16b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-af621 namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 50da2c58-ac14-40a4-ae72-360b9c011d67 kind: Event lastTimestamp: "2026-07-01T18:58:43Z" message: 'Failed to update status for InferenceService "error-404-isvc-af621": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-af621": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:58:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:58:43Z" name: error-404-isvc-af621.18be3f8094e36117 namespace: kserve-ci-e2e-test resourceVersion: "20574" uid: f8881e09-8e9c-4ea3-8313-67cd5ab4951d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-af621 namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 50da2c58-ac14-40a4-ae72-360b9c011d67 kind: Event lastTimestamp: "2026-07-01T18:58:43Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-af621": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:58:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:58:43Z" name: error-404-isvc-af621.18be3f8094e39f69 namespace: kserve-ci-e2e-test resourceVersion: "20575" uid: deb48e87-8f02-4a7f-ad91-a352cab315db reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-af621 namespace: kserve-ci-e2e-test resourceVersion: "21066" uid: 50da2c58-ac14-40a4-ae72-360b9c011d67 kind: Event lastTimestamp: "2026-07-01T18:59:32Z" message: InferenceService [error-404-isvc-af621] is Ready metadata: creationTimestamp: "2026-07-01T18:59:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:59:32Z" name: error-404-isvc-af621.18be3f8be9b81a1d namespace: kserve-ci-e2e-test resourceVersion: "21067" uid: 8cbc9e4f-883d-4666-a9bb-b8a9c8dd8a3b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:57:33.734228Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c059b-predictor-847bfb4858-gvm44 namespace: kserve-ci-e2e-test resourceVersion: "19680" uid: 8000bfc6-d3d5-4f73-b381-69e464c77c0d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c059b-predictor-847bfb4858-gvm44 to ip-10-0-129-245.ec2.internal metadata: creationTimestamp: "2026-07-01T18:57:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:57:33Z" name: error-404-isvc-c059b-predictor-847bfb4858-gvm44.18be3f703a6a6c7c namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: 303383b1-bb8e-494b-88dc-2bc40e25624d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:33Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c059b-predictor-847bfb4858-gvm44 namespace: kserve-ci-e2e-test resourceVersion: "19689" uid: 8000bfc6-d3d5-4f73-b381-69e464c77c0d kind: Event lastTimestamp: "2026-07-01T18:57:33Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-c059b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:57:33Z" name: error-404-isvc-c059b-predictor-847bfb4858-gvm44.18be3f70471d102e namespace: kserve-ci-e2e-test resourceVersion: "19715" uid: aa8776bd-5a76-482f-9a9c-3ee8dd6a6402 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:34Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c059b-predictor-847bfb4858-gvm44 namespace: kserve-ci-e2e-test resourceVersion: "19701" uid: 8000bfc6-d3d5-4f73-b381-69e464c77c0d kind: Event lastTimestamp: "2026-07-01T18:57:34Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:57:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:57:34Z" name: error-404-isvc-c059b-predictor-847bfb4858-gvm44.18be3f70785506ed namespace: kserve-ci-e2e-test resourceVersion: "19729" uid: 26ff5cca-29af-4cc2-bf2e-af083d4b27c0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c059b-predictor-847bfb4858-gvm44 namespace: kserve-ci-e2e-test resourceVersion: "19689" uid: 8000bfc6-d3d5-4f73-b381-69e464c77c0d kind: Event lastTimestamp: "2026-07-01T18:57:34Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T18:57:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:57:34Z" name: error-404-isvc-c059b-predictor-847bfb4858-gvm44.18be3f7079e70dcf namespace: kserve-ci-e2e-test resourceVersion: "19732" uid: 0232a835-bd8e-46f5-9217-093523ef8130 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c059b-predictor-847bfb4858-gvm44 namespace: kserve-ci-e2e-test resourceVersion: "19689" uid: 8000bfc6-d3d5-4f73-b381-69e464c77c0d kind: Event lastTimestamp: "2026-07-01T18:57:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:57:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:57:34Z" name: error-404-isvc-c059b-predictor-847bfb4858-gvm44.18be3f707d15f802 namespace: kserve-ci-e2e-test resourceVersion: "19736" uid: e10fdd94-c0af-4133-aa53-0711129b365a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c059b-predictor-847bfb4858-gvm44 namespace: kserve-ci-e2e-test resourceVersion: "19689" uid: 8000bfc6-d3d5-4f73-b381-69e464c77c0d kind: Event lastTimestamp: "2026-07-01T18:57:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:57:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:57:34Z" name: error-404-isvc-c059b-predictor-847bfb4858-gvm44.18be3f707d9c55a4 namespace: kserve-ci-e2e-test resourceVersion: "19738" uid: b633c901-52d0-40b1-b74b-2d86a15f66ef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c059b-predictor-847bfb4858-gvm44 namespace: kserve-ci-e2e-test resourceVersion: "19689" uid: 8000bfc6-d3d5-4f73-b381-69e464c77c0d kind: Event lastTimestamp: "2026-07-01T18:57:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:57:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:57:34Z" name: error-404-isvc-c059b-predictor-847bfb4858-gvm44.18be3f707da639e4 namespace: kserve-ci-e2e-test resourceVersion: "19739" uid: f16f117a-948c-47b2-9283-2ebb1b1416aa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c059b-predictor-847bfb4858-gvm44 namespace: kserve-ci-e2e-test resourceVersion: "19689" uid: 8000bfc6-d3d5-4f73-b381-69e464c77c0d kind: Event lastTimestamp: "2026-07-01T18:57:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:57:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:57:34Z" name: error-404-isvc-c059b-predictor-847bfb4858-gvm44.18be3f70800ee911 namespace: kserve-ci-e2e-test resourceVersion: "19740" uid: 26738f09-d47b-4b79-92e5-90c805653464 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c059b-predictor-847bfb4858-gvm44 namespace: kserve-ci-e2e-test resourceVersion: "19689" uid: 8000bfc6-d3d5-4f73-b381-69e464c77c0d kind: Event lastTimestamp: "2026-07-01T18:57:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:57:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:57:34Z" name: error-404-isvc-c059b-predictor-847bfb4858-gvm44.18be3f70807a385c namespace: kserve-ci-e2e-test resourceVersion: "19741" uid: f6c5539f-1852-4f59-b153-81c3b5b4f925 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:57:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c059b-predictor-847bfb4858-gvm44 namespace: kserve-ci-e2e-test resourceVersion: "19689" uid: 8000bfc6-d3d5-4f73-b381-69e464c77c0d kind: Event lastTimestamp: "2026-07-01T18:58:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:57:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:12Z" name: error-404-isvc-c059b-predictor-847bfb4858-gvm44.18be3f70c9b3353e namespace: kserve-ci-e2e-test resourceVersion: "20044" uid: 6f4cdbfb-76cd-4f5a-a9ff-08d8a8e039ae reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c059b-predictor-847bfb4858-gvm44 namespace: kserve-ci-e2e-test resourceVersion: "19689" uid: 8000bfc6-d3d5-4f73-b381-69e464c77c0d kind: Event lastTimestamp: "2026-07-01T18:58:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:58:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:43Z" name: error-404-isvc-c059b-predictor-847bfb4858-gvm44.18be3f808c9f307e namespace: kserve-ci-e2e-test resourceVersion: "20521" uid: 0be345d2-a491-4800-9c28-d08dabbb129e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c059b-predictor-847bfb4858-gvm44 namespace: kserve-ci-e2e-test resourceVersion: "19689" uid: 8000bfc6-d3d5-4f73-b381-69e464c77c0d kind: Event lastTimestamp: "2026-07-01T18:58:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:58:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:43Z" name: error-404-isvc-c059b-predictor-847bfb4858-gvm44.18be3f808c9fbb51 namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: 294e753b-f040-4e3b-8564-ebcf463acfab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c059b-predictor-847bfb4858-gvm44 namespace: kserve-ci-e2e-test resourceVersion: "19689" uid: 8000bfc6-d3d5-4f73-b381-69e464c77c0d kind: Event lastTimestamp: "2026-07-01T18:58:47Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:58:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:47Z" name: error-404-isvc-c059b-predictor-847bfb4858-gvm44.18be3f815218ac38 namespace: kserve-ci-e2e-test resourceVersion: "20657" uid: fae0b6cd-6365-4d1c-84ff-955a297fff42 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c059b-predictor-847bfb4858 namespace: kserve-ci-e2e-test resourceVersion: "19670" uid: ff0ea850-e309-41ca-8e00-af99b1954c07 kind: Event lastTimestamp: "2026-07-01T18:57:33Z" message: 'Created pod: error-404-isvc-c059b-predictor-847bfb4858-gvm44' metadata: creationTimestamp: "2026-07-01T18:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:57:33Z" name: error-404-isvc-c059b-predictor-847bfb4858.18be3f70399f2909 namespace: kserve-ci-e2e-test resourceVersion: "19686" uid: b43d4ca9-584b-46c5-a4d4-1ad77db4797c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c059b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19668" uid: 2995faef-2e66-4d3b-9c1e-f7ee5e95df93 kind: Event lastTimestamp: "2026-07-01T18:57:33Z" message: Scaled up replica set error-404-isvc-c059b-predictor-847bfb4858 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:57:33Z" name: error-404-isvc-c059b-predictor.18be3f703769b7f4 namespace: kserve-ci-e2e-test resourceVersion: "19672" uid: ecc9250d-6c3a-49ca-80de-4888fa152a9a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:57:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c059b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19669" uid: 2adc424a-1732-42bb-a2c2-b5820008cf11 kind: Event lastTimestamp: "2026-07-01T18:58:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:57:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:58:03Z" name: error-404-isvc-c059b-predictor.18be3f73b5ffe6e7 namespace: kserve-ci-e2e-test resourceVersion: "19975" uid: 5e62d298-d50a-4f58-a2df-53506942047e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:57:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c059b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19669" uid: 2adc424a-1732-42bb-a2c2-b5820008cf11 kind: Event lastTimestamp: "2026-07-01T18:58:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:57:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:58:03Z" name: error-404-isvc-c059b-predictor.18be3f73b65159d7 namespace: kserve-ci-e2e-test resourceVersion: "19976" uid: e626ac3d-de2a-458b-961e-77fbb10da040 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:58:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c059b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19867" uid: 2adc424a-1732-42bb-a2c2-b5820008cf11 kind: Event lastTimestamp: "2026-07-01T18:58:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:58:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:58:33Z" name: error-404-isvc-c059b-predictor.18be3f7ab3e8a18c namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: 4ba2028d-2679-4273-a7b5-4140ef99dc21 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:58:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c059b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19867" uid: 2adc424a-1732-42bb-a2c2-b5820008cf11 kind: Event lastTimestamp: "2026-07-01T18:58:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:58:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:58:33Z" name: error-404-isvc-c059b-predictor.18be3f7ab458fa27 namespace: kserve-ci-e2e-test resourceVersion: "20389" uid: 90f8fa4b-eb9d-430e-9651-d02e6828f8a2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:57:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c059b namespace: kserve-ci-e2e-test resourceVersion: "19638" uid: f9211c33-8a55-4326-8fd5-a9d0b17acad3 kind: Event lastTimestamp: "2026-07-01T18:58:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:58:22Z" name: error-404-isvc-c059b.18be3f703797e783 namespace: kserve-ci-e2e-test resourceVersion: "20137" uid: 2be91f2c-5bc1-488f-9ab5-596f1cbccd84 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c059b namespace: kserve-ci-e2e-test resourceVersion: "19638" uid: f9211c33-8a55-4326-8fd5-a9d0b17acad3 kind: Event lastTimestamp: "2026-07-01T18:57:33Z" message: 'Failed to update status for InferenceService "error-404-isvc-c059b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c059b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:57:33Z" name: error-404-isvc-c059b.18be3f7038164ac0 namespace: kserve-ci-e2e-test resourceVersion: "19675" uid: 8de56772-4933-4427-9837-ee2b68038df4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c059b namespace: kserve-ci-e2e-test resourceVersion: "19638" uid: f9211c33-8a55-4326-8fd5-a9d0b17acad3 kind: Event lastTimestamp: "2026-07-01T18:57:33Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c059b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:57:33Z" name: error-404-isvc-c059b.18be3f703816af93 namespace: kserve-ci-e2e-test resourceVersion: "19677" uid: bce508ac-0f7a-4b6d-a9df-86805d20a79f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c059b namespace: kserve-ci-e2e-test resourceVersion: "20135" uid: f9211c33-8a55-4326-8fd5-a9d0b17acad3 kind: Event lastTimestamp: "2026-07-01T18:58:22Z" message: InferenceService [error-404-isvc-c059b] is Ready metadata: creationTimestamp: "2026-07-01T18:58:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:58:22Z" name: error-404-isvc-c059b.18be3f7b86269224 namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: ce11a3e9-5902-45a6-a258-8904a6c325d5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:59:53.924527Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c3262-predictor-6676bfcd4d-z54kx namespace: kserve-ci-e2e-test resourceVersion: "21422" uid: ad795212-732d-4487-8321-0cb33464f64f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c3262-predictor-6676bfcd4d-z54kx to ip-10-0-129-245.ec2.internal metadata: creationTimestamp: "2026-07-01T18:59:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:59:53Z" name: error-404-isvc-c3262-predictor-6676bfcd4d-z54kx.18be3f90de68a634 namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: c25ba510-d86c-4b4a-a658-301041a3701d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:54Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c3262-predictor-6676bfcd4d-z54kx namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: ad795212-732d-4487-8321-0cb33464f64f kind: Event lastTimestamp: "2026-07-01T18:59:54Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:59:54Z" name: error-404-isvc-c3262-predictor-6676bfcd4d-z54kx.18be3f90f7e300c1 namespace: kserve-ci-e2e-test resourceVersion: "21446" uid: d545a8e9-1646-4527-be4e-3a764847fd62 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3262-predictor-6676bfcd4d-z54kx namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: ad795212-732d-4487-8321-0cb33464f64f kind: Event lastTimestamp: "2026-07-01T18:59:54Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T18:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:54Z" name: error-404-isvc-c3262-predictor-6676bfcd4d-z54kx.18be3f90f9551c54 namespace: kserve-ci-e2e-test resourceVersion: "21448" uid: 2d34803a-8230-48af-a945-a7a1bed58a13 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3262-predictor-6676bfcd4d-z54kx namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: ad795212-732d-4487-8321-0cb33464f64f kind: Event lastTimestamp: "2026-07-01T18:59:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:54Z" name: error-404-isvc-c3262-predictor-6676bfcd4d-z54kx.18be3f90fc74b950 namespace: kserve-ci-e2e-test resourceVersion: "21449" uid: 812ce36b-b247-4fca-9cfd-c30caf540cb9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3262-predictor-6676bfcd4d-z54kx namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: ad795212-732d-4487-8321-0cb33464f64f kind: Event lastTimestamp: "2026-07-01T18:59:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:54Z" name: error-404-isvc-c3262-predictor-6676bfcd4d-z54kx.18be3f90fce44332 namespace: kserve-ci-e2e-test resourceVersion: "21450" uid: 2d382fe2-4fd2-4d2b-97b4-fee31c5cc569 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3262-predictor-6676bfcd4d-z54kx namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: ad795212-732d-4487-8321-0cb33464f64f kind: Event lastTimestamp: "2026-07-01T18:59:54Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:54Z" name: error-404-isvc-c3262-predictor-6676bfcd4d-z54kx.18be3f90fceeeee1 namespace: kserve-ci-e2e-test resourceVersion: "21451" uid: 417e32c2-d188-4401-ae95-5187801f594d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3262-predictor-6676bfcd4d-z54kx namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: ad795212-732d-4487-8321-0cb33464f64f kind: Event lastTimestamp: "2026-07-01T18:59:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:54Z" name: error-404-isvc-c3262-predictor-6676bfcd4d-z54kx.18be3f90ff7fb356 namespace: kserve-ci-e2e-test resourceVersion: "21452" uid: eeafeff6-e889-4e88-8ba8-2bc9d992a640 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3262-predictor-6676bfcd4d-z54kx namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: ad795212-732d-4487-8321-0cb33464f64f kind: Event lastTimestamp: "2026-07-01T18:59:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:54Z" name: error-404-isvc-c3262-predictor-6676bfcd4d-z54kx.18be3f91000f23fa namespace: kserve-ci-e2e-test resourceVersion: "21453" uid: 87ad6013-c5d6-4acb-847a-83ddb9a8b917 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T18:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3262-predictor-6676bfcd4d-z54kx namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: ad795212-732d-4487-8321-0cb33464f64f kind: Event lastTimestamp: "2026-07-01T19:09:10Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:10Z" name: error-404-isvc-c3262-predictor-6676bfcd4d-z54kx.18be3f910f30c9a5 namespace: kserve-ci-e2e-test resourceVersion: "26197" uid: 09476fa3-4170-4e1a-ba74-67341ef19029 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3262-predictor-6676bfcd4d-z54kx namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: ad795212-732d-4487-8321-0cb33464f64f kind: Event lastTimestamp: "2026-07-01T19:09:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:09:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:08Z" name: error-404-isvc-c3262-predictor-6676bfcd4d-z54kx.18be40120a2a973c namespace: kserve-ci-e2e-test resourceVersion: "26002" uid: 180435ed-0fac-4cae-b436-f20b50973755 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3262-predictor-6676bfcd4d-z54kx namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: ad795212-732d-4487-8321-0cb33464f64f kind: Event lastTimestamp: "2026-07-01T19:09:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:09:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:08Z" name: error-404-isvc-c3262-predictor-6676bfcd4d-z54kx.18be40120a2abc9d namespace: kserve-ci-e2e-test resourceVersion: "26006" uid: 54a706e1-0ae6-4cf8-85ed-e4d1bba1a69b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3262-predictor-6676bfcd4d-z54kx namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: ad795212-732d-4487-8321-0cb33464f64f kind: Event lastTimestamp: "2026-07-01T19:09:10Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:10Z" name: error-404-isvc-c3262-predictor-6676bfcd4d-z54kx.18be401283cd5fc4 namespace: kserve-ci-e2e-test resourceVersion: "26196" uid: b094e9e8-8240-41ed-b978-c214c52b51b7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c3262-predictor-6676bfcd4d namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: f87b634b-4fae-4345-a7a2-79973bb8d9b7 kind: Event lastTimestamp: "2026-07-01T18:59:53Z" message: 'Created pod: error-404-isvc-c3262-predictor-6676bfcd4d-z54kx' metadata: creationTimestamp: "2026-07-01T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:59:53Z" name: error-404-isvc-c3262-predictor-6676bfcd4d.18be3f90ddf285db namespace: kserve-ci-e2e-test resourceVersion: "21423" uid: b5ffaaac-f323-4dcb-a429-cf2b65d9e009 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c3262-predictor namespace: kserve-ci-e2e-test resourceVersion: "21410" uid: 057cf0f5-ce94-44ee-a02a-0a70eac845ef kind: Event lastTimestamp: "2026-07-01T18:59:53Z" message: Scaled up replica set error-404-isvc-c3262-predictor-6676bfcd4d from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:59:53Z" name: error-404-isvc-c3262-predictor.18be3f90dc1f3f05 namespace: kserve-ci-e2e-test resourceVersion: "21415" uid: 48d05c0c-aa00-4635-a620-eab0bdfb8cdd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:00:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3262-predictor namespace: kserve-ci-e2e-test resourceVersion: "21413" uid: e0d9cab7-8da7-4f71-82bb-fdc91bc3c389 kind: Event lastTimestamp: "2026-07-01T19:00:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:00:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:00:53Z" name: error-404-isvc-c3262-predictor.18be3f945b7ffd1b namespace: kserve-ci-e2e-test resourceVersion: "22007" uid: 9f64aba3-ef58-4712-bcc7-d9fef542ad36 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:00:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3262-predictor namespace: kserve-ci-e2e-test resourceVersion: "21413" uid: e0d9cab7-8da7-4f71-82bb-fdc91bc3c389 kind: Event lastTimestamp: "2026-07-01T19:00:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:00:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:00:53Z" name: error-404-isvc-c3262-predictor.18be3f945c176de7 namespace: kserve-ci-e2e-test resourceVersion: "22008" uid: da2571db-6f2d-4b5a-af51-b434b7fbfffb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:59:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3262 namespace: kserve-ci-e2e-test resourceVersion: "21383" uid: 4983368f-703d-4d82-bb9b-fa868f56a308 kind: Event lastTimestamp: "2026-07-01T19:00:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:00:40Z" name: error-404-isvc-c3262.18be3f90dc39cc5a namespace: kserve-ci-e2e-test resourceVersion: "21877" uid: 17145811-03a2-457d-a8eb-84ee90461ec9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3262 namespace: kserve-ci-e2e-test resourceVersion: "21383" uid: 4983368f-703d-4d82-bb9b-fa868f56a308 kind: Event lastTimestamp: "2026-07-01T18:59:53Z" message: 'Failed to update status for InferenceService "error-404-isvc-c3262": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c3262": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:59:53Z" name: error-404-isvc-c3262.18be3f90dcda90c5 namespace: kserve-ci-e2e-test resourceVersion: "21418" uid: 58bceeee-6f13-45ff-a664-7c1dfb1816aa reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3262 namespace: kserve-ci-e2e-test resourceVersion: "21383" uid: 4983368f-703d-4d82-bb9b-fa868f56a308 kind: Event lastTimestamp: "2026-07-01T18:59:53Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c3262": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:59:53Z" name: error-404-isvc-c3262.18be3f90dcdaecbd namespace: kserve-ci-e2e-test resourceVersion: "21420" uid: 22f49368-3c83-4408-a90d-ccee6e4deacf reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:00:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3262 namespace: kserve-ci-e2e-test resourceVersion: "21875" uid: 4983368f-703d-4d82-bb9b-fa868f56a308 kind: Event lastTimestamp: "2026-07-01T19:00:40Z" message: InferenceService [error-404-isvc-c3262] is Ready metadata: creationTimestamp: "2026-07-01T19:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:00:40Z" name: error-404-isvc-c3262.18be3f9bcbd988b9 namespace: kserve-ci-e2e-test resourceVersion: "21876" uid: 98ba4350-e4c3-41c9-a503-153cdb5da867 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:49:14.825099Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-eb211-predictor-6695c85886-hvd2p namespace: kserve-ci-e2e-test resourceVersion: "15955" uid: 8265b84a-2ed0-4bd9-a163-6cd2f9c1b9d0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-eb211-predictor-6695c85886-hvd2p to ip-10-0-129-245.ec2.internal metadata: creationTimestamp: "2026-07-01T18:49:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:49:14Z" name: error-404-isvc-eb211-predictor-6695c85886-hvd2p.18be3efc111d4a39 namespace: kserve-ci-e2e-test resourceVersion: "15966" uid: d19eff94-bab4-4922-aa16-af8e6740f7aa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:15Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-eb211-predictor-6695c85886-hvd2p namespace: kserve-ci-e2e-test resourceVersion: "15960" uid: 8265b84a-2ed0-4bd9-a163-6cd2f9c1b9d0 kind: Event lastTimestamp: "2026-07-01T18:49:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-eb211-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:15Z" name: error-404-isvc-eb211-predictor-6695c85886-hvd2p.18be3efc1c86c601 namespace: kserve-ci-e2e-test resourceVersion: "15986" uid: ed9f4fc3-e986-495d-a87c-3e0712172d64 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:15Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-eb211-predictor-6695c85886-hvd2p namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: 8265b84a-2ed0-4bd9-a163-6cd2f9c1b9d0 kind: Event lastTimestamp: "2026-07-01T18:49:15Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:49:15Z" name: error-404-isvc-eb211-predictor-6695c85886-hvd2p.18be3efc4f0f26e2 namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 8cfcaa39-40a8-4f55-b0ae-e73b263d07f2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-eb211-predictor-6695c85886-hvd2p namespace: kserve-ci-e2e-test resourceVersion: "15960" uid: 8265b84a-2ed0-4bd9-a163-6cd2f9c1b9d0 kind: Event lastTimestamp: "2026-07-01T18:49:15Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T18:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:15Z" name: error-404-isvc-eb211-predictor-6695c85886-hvd2p.18be3efc507bc91c namespace: kserve-ci-e2e-test resourceVersion: "16007" uid: 5be77078-ccb7-4e42-91fc-6fb30c3c98a4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-eb211-predictor-6695c85886-hvd2p namespace: kserve-ci-e2e-test resourceVersion: "15960" uid: 8265b84a-2ed0-4bd9-a163-6cd2f9c1b9d0 kind: Event lastTimestamp: "2026-07-01T18:49:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:15Z" name: error-404-isvc-eb211-predictor-6695c85886-hvd2p.18be3efc53a95bd0 namespace: kserve-ci-e2e-test resourceVersion: "16010" uid: 9b5ea978-1251-4149-afed-8e5f06b71f9f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-eb211-predictor-6695c85886-hvd2p namespace: kserve-ci-e2e-test resourceVersion: "15960" uid: 8265b84a-2ed0-4bd9-a163-6cd2f9c1b9d0 kind: Event lastTimestamp: "2026-07-01T18:49:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:15Z" name: error-404-isvc-eb211-predictor-6695c85886-hvd2p.18be3efc541ce243 namespace: kserve-ci-e2e-test resourceVersion: "16011" uid: 56c0069c-a59e-4549-9e6c-6c4aaf7317c2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-eb211-predictor-6695c85886-hvd2p namespace: kserve-ci-e2e-test resourceVersion: "15960" uid: 8265b84a-2ed0-4bd9-a163-6cd2f9c1b9d0 kind: Event lastTimestamp: "2026-07-01T18:49:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:15Z" name: error-404-isvc-eb211-predictor-6695c85886-hvd2p.18be3efc54267cbc namespace: kserve-ci-e2e-test resourceVersion: "16012" uid: edf54645-d3e1-489b-8dbe-c2c6d02a8a90 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-eb211-predictor-6695c85886-hvd2p namespace: kserve-ci-e2e-test resourceVersion: "15960" uid: 8265b84a-2ed0-4bd9-a163-6cd2f9c1b9d0 kind: Event lastTimestamp: "2026-07-01T18:49:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:16Z" name: error-404-isvc-eb211-predictor-6695c85886-hvd2p.18be3efc5733c888 namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: c092ceca-d3ff-406d-94f5-aca2047eda66 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-eb211-predictor-6695c85886-hvd2p namespace: kserve-ci-e2e-test resourceVersion: "15960" uid: 8265b84a-2ed0-4bd9-a163-6cd2f9c1b9d0 kind: Event lastTimestamp: "2026-07-01T18:49:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:16Z" name: error-404-isvc-eb211-predictor-6695c85886-hvd2p.18be3efc579edb38 namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: ef6f2a8e-a8ef-4ebe-966d-afb3ecc3bc9a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T18:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-eb211-predictor-6695c85886-hvd2p namespace: kserve-ci-e2e-test resourceVersion: "15960" uid: 8265b84a-2ed0-4bd9-a163-6cd2f9c1b9d0 kind: Event lastTimestamp: "2026-07-01T18:58:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:32Z" name: error-404-isvc-eb211-predictor-6695c85886-hvd2p.18be3efc6792bf54 namespace: kserve-ci-e2e-test resourceVersion: "20345" uid: fd5b115f-6a69-4d86-a8d9-ba1497dcd7b2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-eb211-predictor-6695c85886-hvd2p namespace: kserve-ci-e2e-test resourceVersion: "15960" uid: 8265b84a-2ed0-4bd9-a163-6cd2f9c1b9d0 kind: Event lastTimestamp: "2026-07-01T18:58:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:29Z" name: error-404-isvc-eb211-predictor-6695c85886-hvd2p.18be3f7d2ff431d5 namespace: kserve-ci-e2e-test resourceVersion: "20217" uid: 3ec86e7e-5209-40aa-afcf-897b28be1900 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-eb211-predictor-6695c85886-hvd2p namespace: kserve-ci-e2e-test resourceVersion: "15960" uid: 8265b84a-2ed0-4bd9-a163-6cd2f9c1b9d0 kind: Event lastTimestamp: "2026-07-01T18:58:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:29Z" name: error-404-isvc-eb211-predictor-6695c85886-hvd2p.18be3f7d2ff48a8a namespace: kserve-ci-e2e-test resourceVersion: "20219" uid: 4d68f142-47f3-4fbb-a491-99a24dd6ed52 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-eb211-predictor-6695c85886-hvd2p namespace: kserve-ci-e2e-test resourceVersion: "15960" uid: 8265b84a-2ed0-4bd9-a163-6cd2f9c1b9d0 kind: Event lastTimestamp: "2026-07-01T18:58:32Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:58:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:32Z" name: error-404-isvc-eb211-predictor-6695c85886-hvd2p.18be3f7ddc168098 namespace: kserve-ci-e2e-test resourceVersion: "20343" uid: f610dff2-d673-4d00-ac8a-87d5ff8a4c5c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-eb211-predictor-6695c85886 namespace: kserve-ci-e2e-test resourceVersion: "15945" uid: ac7505ca-4900-40e6-88e8-14ebfebc008d kind: Event lastTimestamp: "2026-07-01T18:49:14Z" message: 'Created pod: error-404-isvc-eb211-predictor-6695c85886-hvd2p' metadata: creationTimestamp: "2026-07-01T18:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:49:14Z" name: error-404-isvc-eb211-predictor-6695c85886.18be3efc10e27fbf namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 4ce186ab-e697-4c6a-9f92-58a8bb8d6ccf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-eb211-predictor namespace: kserve-ci-e2e-test resourceVersion: "15943" uid: b70b1506-040e-48b0-9dd3-968177a336cf kind: Event lastTimestamp: "2026-07-01T18:49:14Z" message: Scaled up replica set error-404-isvc-eb211-predictor-6695c85886 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:49:14Z" name: error-404-isvc-eb211-predictor.18be3efc0e5e54ea namespace: kserve-ci-e2e-test resourceVersion: "15946" uid: 163bed5c-7ca8-4505-8d51-e3a7368ad8da reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-eb211-predictor namespace: kserve-ci-e2e-test resourceVersion: "15944" uid: f7bb20ae-a86f-4b16-97a3-f82998420033 kind: Event lastTimestamp: "2026-07-01T18:49:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:49:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:49:29Z" name: error-404-isvc-eb211-predictor.18be3eff8cf9207b namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: 157c40ee-df3b-44a0-bdb1-b790be9a04f4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-eb211-predictor namespace: kserve-ci-e2e-test resourceVersion: "15944" uid: f7bb20ae-a86f-4b16-97a3-f82998420033 kind: Event lastTimestamp: "2026-07-01T18:49:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:49:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:49:29Z" name: error-404-isvc-eb211-predictor.18be3eff8d5e933a namespace: kserve-ci-e2e-test resourceVersion: "16192" uid: 71602156-ac58-4b2d-8267-c4eb96e90e5b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:49:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-eb211-predictor namespace: kserve-ci-e2e-test resourceVersion: "16191" uid: f7bb20ae-a86f-4b16-97a3-f82998420033 kind: Event lastTimestamp: "2026-07-01T18:50:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:50:29Z" name: error-404-isvc-eb211-predictor.18be3f030c295d9d namespace: kserve-ci-e2e-test resourceVersion: "16704" uid: c42e624b-48bb-46e6-9c4f-c2d6268efe3f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:49:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-eb211-predictor namespace: kserve-ci-e2e-test resourceVersion: "16191" uid: f7bb20ae-a86f-4b16-97a3-f82998420033 kind: Event lastTimestamp: "2026-07-01T18:50:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:50:29Z" name: error-404-isvc-eb211-predictor.18be3f030c9ec68d namespace: kserve-ci-e2e-test resourceVersion: "16705" uid: bce48541-a7de-4aa8-b046-575e0bccbe11 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:49:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-eb211 namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: 7141a350-8d39-4e2b-9fc7-041e6a7a12eb kind: Event lastTimestamp: "2026-07-01T18:50:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:50:02Z" name: error-404-isvc-eb211.18be3efc0e6125b6 namespace: kserve-ci-e2e-test resourceVersion: "16463" uid: 0e2ba421-41fe-49ed-9bac-04ea61d74e16 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:50:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-eb211 namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: 7141a350-8d39-4e2b-9fc7-041e6a7a12eb kind: Event lastTimestamp: "2026-07-01T18:50:02Z" message: InferenceService [error-404-isvc-eb211] is Ready metadata: creationTimestamp: "2026-07-01T18:50:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:50:02Z" name: error-404-isvc-eb211.18be3f0723dc0815 namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: b73183b6-b87c-4bfb-a3bc-40c4861b7f21 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:46:45.122092Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd namespace: kserve-ci-e2e-test resourceVersion: "14216" uid: 1932b4d7-e389-4605-bda4-4728ecf22790 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd to ip-10-0-129-245.ec2.internal metadata: creationTimestamp: "2026-07-01T18:46:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:46:45Z" name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd.18be3ed9361eb8c2 namespace: kserve-ci-e2e-test resourceVersion: "14220" uid: ae6f7feb-71ae-4cf4-a123-612ec28f0223 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:45Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd namespace: kserve-ci-e2e-test resourceVersion: "14218" uid: 1932b4d7-e389-4605-bda4-4728ecf22790 kind: Event lastTimestamp: "2026-07-01T18:46:45Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-f2cdf-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:46:45Z" name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd.18be3ed93d318e01 namespace: kserve-ci-e2e-test resourceVersion: "14241" uid: 2bfc6c27-d42f-4f19-9af2-9c45afb7a14d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd namespace: kserve-ci-e2e-test resourceVersion: "14223" uid: 1932b4d7-e389-4605-bda4-4728ecf22790 kind: Event lastTimestamp: "2026-07-01T18:46:46Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:46:46Z" name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd.18be3ed97744e51d namespace: kserve-ci-e2e-test resourceVersion: "14289" uid: 8396dda4-9b55-4f55-be2e-9b9f6722222a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd namespace: kserve-ci-e2e-test resourceVersion: "14218" uid: 1932b4d7-e389-4605-bda4-4728ecf22790 kind: Event lastTimestamp: "2026-07-01T18:46:46Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" metadata: creationTimestamp: "2026-07-01T18:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:46:46Z" name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd.18be3ed9797b5990 namespace: kserve-ci-e2e-test resourceVersion: "14291" uid: 51f4137f-1fc9-4dab-9680-a48784851c4d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd namespace: kserve-ci-e2e-test resourceVersion: "14218" uid: 1932b4d7-e389-4605-bda4-4728ecf22790 kind: Event lastTimestamp: "2026-07-01T18:47:01Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1690" in 15.621s (15.621s including waiting). Image size: 1334589547 bytes.' metadata: creationTimestamp: "2026-07-01T18:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:01Z" name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd.18be3edd1c98729b namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: c3963e11-4f7c-49d4-b248-5c4ecde0e9e9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd namespace: kserve-ci-e2e-test resourceVersion: "14218" uid: 1932b4d7-e389-4605-bda4-4728ecf22790 kind: Event lastTimestamp: "2026-07-01T18:47:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:01Z" name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd.18be3edd20898748 namespace: kserve-ci-e2e-test resourceVersion: "14462" uid: f5d9ff34-90bb-4bf6-bd29-63d0460bd7aa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd namespace: kserve-ci-e2e-test resourceVersion: "14218" uid: 1932b4d7-e389-4605-bda4-4728ecf22790 kind: Event lastTimestamp: "2026-07-01T18:47:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:01Z" name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd.18be3edd211795a1 namespace: kserve-ci-e2e-test resourceVersion: "14463" uid: 45c7ae02-c609-415c-9bc9-416d08cbf7b7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd namespace: kserve-ci-e2e-test resourceVersion: "14218" uid: 1932b4d7-e389-4605-bda4-4728ecf22790 kind: Event lastTimestamp: "2026-07-01T18:47:01Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T18:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:01Z" name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd.18be3edd212680c6 namespace: kserve-ci-e2e-test resourceVersion: "14466" uid: 4e0d9317-7fcb-4041-be0e-756b0739a19d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd namespace: kserve-ci-e2e-test resourceVersion: "14218" uid: 1932b4d7-e389-4605-bda4-4728ecf22790 kind: Event lastTimestamp: "2026-07-01T18:47:04Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.274s (2.274s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T18:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:04Z" name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd.18be3edda8c01e6b namespace: kserve-ci-e2e-test resourceVersion: "14482" uid: 2fe2873c-7df7-44f3-a8cd-393825c9523f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd namespace: kserve-ci-e2e-test resourceVersion: "14218" uid: 1932b4d7-e389-4605-bda4-4728ecf22790 kind: Event lastTimestamp: "2026-07-01T18:47:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:04Z" name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd.18be3eddac6024b8 namespace: kserve-ci-e2e-test resourceVersion: "14484" uid: f62dea1d-9e6f-4400-a674-0ef1a001cbb6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd namespace: kserve-ci-e2e-test resourceVersion: "14218" uid: 1932b4d7-e389-4605-bda4-4728ecf22790 kind: Event lastTimestamp: "2026-07-01T18:47:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:04Z" name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd.18be3eddacd8d9ec namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: 5895125d-713f-4cd8-b866-43ed86fdf597 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T18:47:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd namespace: kserve-ci-e2e-test resourceVersion: "14218" uid: 1932b4d7-e389-4605-bda4-4728ecf22790 kind: Event lastTimestamp: "2026-07-01T18:48:21Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:47:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:21Z" name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd.18be3ede04f68e2d namespace: kserve-ci-e2e-test resourceVersion: "15349" uid: 7b30411d-1c0c-41e5-b2a0-3831fb4a598b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd namespace: kserve-ci-e2e-test resourceVersion: "14218" uid: 1932b4d7-e389-4605-bda4-4728ecf22790 kind: Event lastTimestamp: "2026-07-01T18:48:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:18Z" name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd.18be3eef0fc53308 namespace: kserve-ci-e2e-test resourceVersion: "15287" uid: fd84d7cf-ef5a-4ffb-98ab-9d9eeaa888ce reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd namespace: kserve-ci-e2e-test resourceVersion: "14218" uid: 1932b4d7-e389-4605-bda4-4728ecf22790 kind: Event lastTimestamp: "2026-07-01T18:48:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:18Z" name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd.18be3eef0fccac56 namespace: kserve-ci-e2e-test resourceVersion: "15290" uid: 73cef0f4-27a6-4629-947c-8ff47d65aa54 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd namespace: kserve-ci-e2e-test resourceVersion: "14218" uid: 1932b4d7-e389-4605-bda4-4728ecf22790 kind: Event lastTimestamp: "2026-07-01T18:48:21Z" message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp 10.134.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:21Z" name: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd.18be3eefb7216c20 namespace: kserve-ci-e2e-test resourceVersion: "15348" uid: f21badb1-dab0-4f9e-9d88-5eee9e4abc5e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f2cdf-predictor-58c9fc6cc8 namespace: kserve-ci-e2e-test resourceVersion: "14207" uid: 953c4bcf-97ee-48a4-88b3-f63726080159 kind: Event lastTimestamp: "2026-07-01T18:46:45Z" message: 'Created pod: error-404-isvc-f2cdf-predictor-58c9fc6cc8-6spwd' metadata: creationTimestamp: "2026-07-01T18:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:46:45Z" name: error-404-isvc-f2cdf-predictor-58c9fc6cc8.18be3ed935995a01 namespace: kserve-ci-e2e-test resourceVersion: "14217" uid: ecf6923d-b932-458b-ac54-4e8a53cbfff3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f2cdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "14205" uid: c1d925ef-36ba-4592-86f9-cb02b7467a0b kind: Event lastTimestamp: "2026-07-01T18:46:45Z" message: Scaled up replica set error-404-isvc-f2cdf-predictor-58c9fc6cc8 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:46:45Z" name: error-404-isvc-f2cdf-predictor.18be3ed93084fead namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 6b02476e-2f96-4a3a-885c-00cbf25b83cf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:47:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f2cdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "14206" uid: 33b3d49b-a4eb-414f-aa08-390e3e10c70b kind: Event lastTimestamp: "2026-07-01T18:47:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:47:15Z" name: error-404-isvc-f2cdf-predictor.18be3edcaee34b51 namespace: kserve-ci-e2e-test resourceVersion: "14576" uid: 86f6e734-1555-4549-a50e-c1e5f0e78e97 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:47:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f2cdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "14206" uid: 33b3d49b-a4eb-414f-aa08-390e3e10c70b kind: Event lastTimestamp: "2026-07-01T18:47:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:47:15Z" name: error-404-isvc-f2cdf-predictor.18be3edcaf65ef4c namespace: kserve-ci-e2e-test resourceVersion: "14577" uid: ae8fac5a-7c5d-4ba8-979e-a40bda4d04f2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:47:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f2cdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: 33b3d49b-a4eb-414f-aa08-390e3e10c70b kind: Event lastTimestamp: "2026-07-01T18:48:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:48:15Z" name: error-404-isvc-f2cdf-predictor.18be3ee3acfc8856 namespace: kserve-ci-e2e-test resourceVersion: "15152" uid: ac6829ce-1fb3-4924-9c07-f529324ed1dd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:47:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f2cdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: 33b3d49b-a4eb-414f-aa08-390e3e10c70b kind: Event lastTimestamp: "2026-07-01T18:48:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:48:15Z" name: error-404-isvc-f2cdf-predictor.18be3ee3ad69701b namespace: kserve-ci-e2e-test resourceVersion: "15153" uid: b94f77f2-05ca-4164-b0dc-f4d019763486 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:46:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f2cdf namespace: kserve-ci-e2e-test resourceVersion: "14187" uid: 2f951325-50cc-4350-af41-b68cf2caf3b5 kind: Event lastTimestamp: "2026-07-01T18:47:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:47:51Z" name: error-404-isvc-f2cdf.18be3ed930abc48e namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: 0ff323b2-26b8-4fbc-a0a5-6a1c431c14da reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f2cdf namespace: kserve-ci-e2e-test resourceVersion: "14917" uid: 2f951325-50cc-4350-af41-b68cf2caf3b5 kind: Event lastTimestamp: "2026-07-01T18:47:51Z" message: InferenceService [error-404-isvc-f2cdf] is Ready metadata: creationTimestamp: "2026-07-01T18:47:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:47:51Z" name: error-404-isvc-f2cdf.18be3ee8c3ba2cb7 namespace: kserve-ci-e2e-test resourceVersion: "14918" uid: 0a77efc4-85c4-4546-910b-d6d74ea7facc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:46:44.542671Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k namespace: kserve-ci-e2e-test resourceVersion: "14130" uid: 4b8e9b12-0793-48ed-9d3d-1d855f2d81d6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-875bcd459-8wb7k to ip-10-0-134-114.ec2.internal metadata: creationTimestamp: "2026-07-01T18:46:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:46:44Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k.18be3ed9139584b2 namespace: kserve-ci-e2e-test resourceVersion: "14138" uid: f02d9163-41fb-409b-8b8f-64ad0aeeb731 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:44Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: 4b8e9b12-0793-48ed-9d3d-1d855f2d81d6 kind: Event lastTimestamp: "2026-07-01T18:46:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:46:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:46:44Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k.18be3ed921ff7117 namespace: kserve-ci-e2e-test resourceVersion: "14168" uid: b10fb62b-8239-4eca-a747-6ee67faeea4d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:45Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k namespace: kserve-ci-e2e-test resourceVersion: "14141" uid: 4b8e9b12-0793-48ed-9d3d-1d855f2d81d6 kind: Event lastTimestamp: "2026-07-01T18:46:45Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:46:45Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k.18be3ed951024b6d namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: 742ecf2f-9f47-4ab4-b2dd-28b97a3fa408 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: 4b8e9b12-0793-48ed-9d3d-1d855f2d81d6 kind: Event lastTimestamp: "2026-07-01T18:46:45Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5df3316e2bbb1c8de0277d9a0c433b6bf041a14baf6838a11e4f2bdc5ed51f25" already present on machine metadata: creationTimestamp: "2026-07-01T18:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:46:45Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k.18be3ed9541a8b93 namespace: kserve-ci-e2e-test resourceVersion: "14277" uid: e2c2a640-e710-4933-8e1d-9aacb9a6ba6e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: 4b8e9b12-0793-48ed-9d3d-1d855f2d81d6 kind: Event lastTimestamp: "2026-07-01T18:46:45Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T18:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:46:45Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k.18be3ed958126544 namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 5a98d965-1e54-489b-b639-db6397c47627 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: 4b8e9b12-0793-48ed-9d3d-1d855f2d81d6 kind: Event lastTimestamp: "2026-07-01T18:46:45Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T18:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:46:45Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k.18be3ed9588bfa56 namespace: kserve-ci-e2e-test resourceVersion: "14280" uid: 5df97d36-4c68-43af-af9e-fd884b78c8d2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: 4b8e9b12-0793-48ed-9d3d-1d855f2d81d6 kind: Event lastTimestamp: "2026-07-01T18:46:49Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" metadata: creationTimestamp: "2026-07-01T18:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:46:49Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k.18be3eda33d8a05e namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: 67b0738e-c63b-4594-9605-0855e76d17dd reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: 4b8e9b12-0793-48ed-9d3d-1d855f2d81d6 kind: Event lastTimestamp: "2026-07-01T18:47:17Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" in 28.021s (28.021s including waiting). Image size: 1558043571 bytes.' metadata: creationTimestamp: "2026-07-01T18:47:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:17Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k.18be3ee0ba110253 namespace: kserve-ci-e2e-test resourceVersion: "14594" uid: bddfa249-2e13-4ecd-b932-c52ea876d33d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: 4b8e9b12-0793-48ed-9d3d-1d855f2d81d6 kind: Event lastTimestamp: "2026-07-01T18:47:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:47:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:17Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k.18be3ee0c0cd8625 namespace: kserve-ci-e2e-test resourceVersion: "14599" uid: c8f61d86-9075-4321-a2f6-4419dd4df24a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: 4b8e9b12-0793-48ed-9d3d-1d855f2d81d6 kind: Event lastTimestamp: "2026-07-01T18:47:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:47:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:17Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k.18be3ee0c1d9b414 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: ad662a05-14cc-43d1-b9b1-661af37ab162 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: 4b8e9b12-0793-48ed-9d3d-1d855f2d81d6 kind: Event lastTimestamp: "2026-07-01T18:47:17Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T18:47:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:17Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k.18be3ee0c1e734d2 namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: a7d5a299-fb61-4df9-9af6-8af8541688a5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: 4b8e9b12-0793-48ed-9d3d-1d855f2d81d6 kind: Event lastTimestamp: "2026-07-01T18:47:20Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.666s (2.666s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T18:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:20Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k.18be3ee160de21a5 namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: 717ee5d6-0c5a-4958-a8ad-4e16b8414f35 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: 4b8e9b12-0793-48ed-9d3d-1d855f2d81d6 kind: Event lastTimestamp: "2026-07-01T18:47:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:20Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k.18be3ee164eeb279 namespace: kserve-ci-e2e-test resourceVersion: "14632" uid: 0c1268b0-7b1c-4e6d-a1bd-238e788e5db2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: 4b8e9b12-0793-48ed-9d3d-1d855f2d81d6 kind: Event lastTimestamp: "2026-07-01T18:47:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:20Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k.18be3ee165be653e namespace: kserve-ci-e2e-test resourceVersion: "14637" uid: ce011ea7-d65e-4b88-97aa-5e2fc4c13a5e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T18:47:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: 4b8e9b12-0793-48ed-9d3d-1d855f2d81d6 kind: Event lastTimestamp: "2026-07-01T18:49:18Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:47:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:18Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k.18be3ee1b49f9a9f namespace: kserve-ci-e2e-test resourceVersion: "16083" uid: b83da8eb-77d8-4f2f-845f-8dbdccc3920a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: 4b8e9b12-0793-48ed-9d3d-1d855f2d81d6 kind: Event lastTimestamp: "2026-07-01T18:49:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:14Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k.18be3efc0a7235db namespace: kserve-ci-e2e-test resourceVersion: "15932" uid: fdcd863f-49d9-4146-a164-8d7a864fcff1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: 4b8e9b12-0793-48ed-9d3d-1d855f2d81d6 kind: Event lastTimestamp: "2026-07-01T18:49:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:14Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k.18be3efc0a72a23d namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: 686eecc6-b7fb-43f6-a039-0abeb4550c2b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k namespace: kserve-ci-e2e-test resourceVersion: "14134" uid: 4b8e9b12-0793-48ed-9d3d-1d855f2d81d6 kind: Event lastTimestamp: "2026-07-01T18:49:17Z" message: 'Readiness probe failed: Get "https://10.132.0.24:8643/healthz": dial tcp 10.132.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:49:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:17Z" name: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k.18be3efcb6f71d2e namespace: kserve-ci-e2e-test resourceVersion: "16040" uid: 429b3c31-47a3-408a-91b5-a9d55e76b380 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-875bcd459 namespace: kserve-ci-e2e-test resourceVersion: "14121" uid: d31eec0c-d63b-4da0-9aa1-c9525fb0651a kind: Event lastTimestamp: "2026-07-01T18:46:44Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-875bcd459-8wb7k' metadata: creationTimestamp: "2026-07-01T18:46:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:46:44Z" name: isvc-sklearn-graph-1-predictor-875bcd459.18be3ed912c5dcae namespace: kserve-ci-e2e-test resourceVersion: "14133" uid: e4565633-bd68-4381-b1c3-5db09f389de1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14120" uid: 966db643-ee7b-4606-8803-102c93c85a64 kind: Event lastTimestamp: "2026-07-01T18:46:44Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-875bcd459 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:46:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:46:44Z" name: isvc-sklearn-graph-1-predictor.18be3ed90b553b6a namespace: kserve-ci-e2e-test resourceVersion: "14123" uid: bdd33703-77e2-4151-80f9-8cd2f21799fc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:46:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14127" uid: b615659a-8adb-413a-b697-ac0ba33682fd kind: Event lastTimestamp: "2026-07-01T18:47:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:47:44Z" name: isvc-sklearn-graph-1-predictor.18be3edc91789205 namespace: kserve-ci-e2e-test resourceVersion: "14840" uid: 79b4e3d7-26d5-4fe0-874e-b8b38f375683 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:46:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14127" uid: b615659a-8adb-413a-b697-ac0ba33682fd kind: Event lastTimestamp: "2026-07-01T18:47:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:47:44Z" name: isvc-sklearn-graph-1-predictor.18be3edc92060782 namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 695d849b-b32e-4e41-81a9-1cc077bd3dde reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:47:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14426" uid: b615659a-8adb-413a-b697-ac0ba33682fd kind: Event lastTimestamp: "2026-07-01T18:48:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:48:59Z" name: isvc-sklearn-graph-1-predictor.18be3eea8eb4f3e7 namespace: kserve-ci-e2e-test resourceVersion: "15681" uid: 8bbfa2a2-0173-4181-bbd2-ce252bde746e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:47:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14426" uid: b615659a-8adb-413a-b697-ac0ba33682fd kind: Event lastTimestamp: "2026-07-01T18:48:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:48:59Z" name: isvc-sklearn-graph-1-predictor.18be3eea8f2a2da6 namespace: kserve-ci-e2e-test resourceVersion: "15682" uid: 60a43146-d745-4156-a00c-b13b9a9312d6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T18:46:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14110" uid: 01f8a02c-e959-4e96-8024-51d256ef685e kind: Event lastTimestamp: "2026-07-01T18:48:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:46:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:48:39Z" name: isvc-sklearn-graph-1.18be3ed9119c079d namespace: kserve-ci-e2e-test resourceVersion: "15524" uid: 0d9f2c0d-52d0-4d0c-9885-8b69126b345e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15522" uid: 01f8a02c-e959-4e96-8024-51d256ef685e kind: Event lastTimestamp: "2026-07-01T18:48:39Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-01T18:48:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:48:39Z" name: isvc-sklearn-graph-1.18be3ef3bd4dffbf namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: 60748a18-2a3b-46c6-a5ba-9b31d9645dec reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:46:45.287271Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j namespace: kserve-ci-e2e-test resourceVersion: "14242" uid: 9d478b59-375d-4cf4-a291-a1ecc867974e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j to ip-10-0-134-114.ec2.internal metadata: creationTimestamp: "2026-07-01T18:46:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:46:45Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j.18be3ed93ff713c6 namespace: kserve-ci-e2e-test resourceVersion: "14246" uid: 3227aa93-f370-4d9e-bd32-ede1b1b0fb39 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:45Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: 9d478b59-375d-4cf4-a291-a1ecc867974e kind: Event lastTimestamp: "2026-07-01T18:46:45Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:46:45Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j.18be3ed94c2eaee3 namespace: kserve-ci-e2e-test resourceVersion: "14267" uid: 705ece70-622c-4f9f-ba05-6ff153dd884f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:46Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: 9d478b59-375d-4cf4-a291-a1ecc867974e kind: Event lastTimestamp: "2026-07-01T18:46:46Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:46:46Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j.18be3ed97d25e873 namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: bf038889-06ca-44d4-a17f-df6b0fcc4352 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: 9d478b59-375d-4cf4-a291-a1ecc867974e kind: Event lastTimestamp: "2026-07-01T18:46:46Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5df3316e2bbb1c8de0277d9a0c433b6bf041a14baf6838a11e4f2bdc5ed51f25" already present on machine metadata: creationTimestamp: "2026-07-01T18:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:46:46Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j.18be3ed97fbc5fed namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: 74abb62e-d74c-488b-a8bd-1213df77f3a9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: 9d478b59-375d-4cf4-a291-a1ecc867974e kind: Event lastTimestamp: "2026-07-01T18:46:46Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T18:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:46:46Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j.18be3ed98297f080 namespace: kserve-ci-e2e-test resourceVersion: "14299" uid: 22c1dab4-913a-4825-8244-ce5372bd989b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: 9d478b59-375d-4cf4-a291-a1ecc867974e kind: Event lastTimestamp: "2026-07-01T18:46:46Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T18:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:46:46Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j.18be3ed9830aee68 namespace: kserve-ci-e2e-test resourceVersion: "14301" uid: 82a7ba86-fb67-4c82-9653-cf1dd3b0bd23 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: 9d478b59-375d-4cf4-a291-a1ecc867974e kind: Event lastTimestamp: "2026-07-01T18:46:50Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" metadata: creationTimestamp: "2026-07-01T18:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:46:50Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j.18be3eda70793618 namespace: kserve-ci-e2e-test resourceVersion: "14355" uid: 9156c404-c3d4-496d-a131-545bd91688b4 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: 9d478b59-375d-4cf4-a291-a1ecc867974e kind: Event lastTimestamp: "2026-07-01T18:47:18Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" in 27.672s (27.672s including waiting). Image size: 1558043571 bytes.' metadata: creationTimestamp: "2026-07-01T18:47:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:18Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j.18be3ee0e1e6d116 namespace: kserve-ci-e2e-test resourceVersion: "14610" uid: 929254a6-50ac-4b08-92ff-e543cab54d66 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: 9d478b59-375d-4cf4-a291-a1ecc867974e kind: Event lastTimestamp: "2026-07-01T18:47:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:47:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:18Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j.18be3ee0e8327e25 namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: 2b63ce39-65f2-4018-8cea-97aae34bc8ad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: 9d478b59-375d-4cf4-a291-a1ecc867974e kind: Event lastTimestamp: "2026-07-01T18:47:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:47:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:18Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j.18be3ee0e98116ed namespace: kserve-ci-e2e-test resourceVersion: "14614" uid: e86cadbb-3e59-4d93-b4b1-d1b046ae9bef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: 9d478b59-375d-4cf4-a291-a1ecc867974e kind: Event lastTimestamp: "2026-07-01T18:47:18Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T18:47:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:18Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j.18be3ee0e992c8ac namespace: kserve-ci-e2e-test resourceVersion: "14615" uid: 5f014aa7-06aa-4815-afae-9d77086d46af reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: 9d478b59-375d-4cf4-a291-a1ecc867974e kind: Event lastTimestamp: "2026-07-01T18:47:20Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.019s (2.019s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T18:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:20Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j.18be3ee161f7461b namespace: kserve-ci-e2e-test resourceVersion: "14631" uid: 72a8dc2e-fa15-4a35-9475-436584421c03 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: 9d478b59-375d-4cf4-a291-a1ecc867974e kind: Event lastTimestamp: "2026-07-01T18:47:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:20Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j.18be3ee16524b76a namespace: kserve-ci-e2e-test resourceVersion: "14633" uid: 7378305d-8881-452e-a25a-3cdb09d68ee8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: 9d478b59-375d-4cf4-a291-a1ecc867974e kind: Event lastTimestamp: "2026-07-01T18:47:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:20Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j.18be3ee1659779ab namespace: kserve-ci-e2e-test resourceVersion: "14635" uid: 6d542287-fa63-4878-9c7c-a323ed321beb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T18:47:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: 9d478b59-375d-4cf4-a291-a1ecc867974e kind: Event lastTimestamp: "2026-07-01T18:49:16Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:16Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j.18be3ee1789e3627 namespace: kserve-ci-e2e-test resourceVersion: "16032" uid: 262a3afe-44c1-49c6-87d8-9ffe3f29e150 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: 9d478b59-375d-4cf4-a291-a1ecc867974e kind: Event lastTimestamp: "2026-07-01T18:49:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:14Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j.18be3efc05c72553 namespace: kserve-ci-e2e-test resourceVersion: "15902" uid: fe87a415-1a5e-4ba1-bd4a-52c191afd9b8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: 9d478b59-375d-4cf4-a291-a1ecc867974e kind: Event lastTimestamp: "2026-07-01T18:49:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:14Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j.18be3efc05c78fac namespace: kserve-ci-e2e-test resourceVersion: "15905" uid: 307d0928-a9bb-4b47-9c7f-25306c248769 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: 9d478b59-375d-4cf4-a291-a1ecc867974e kind: Event lastTimestamp: "2026-07-01T18:49:16Z" message: 'Readiness probe failed: Get "https://10.132.0.26:8643/healthz": dial tcp 10.132.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:16Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j.18be3efc7b204f3e namespace: kserve-ci-e2e-test resourceVersion: "16030" uid: 5bfd3070-cba6-4647-9835-6beb8af8f018 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-6c8478f9b5 namespace: kserve-ci-e2e-test resourceVersion: "14232" uid: d537e5af-97f8-4fea-ab1e-1254c0cf1c1c kind: Event lastTimestamp: "2026-07-01T18:46:45Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-6c8478f9b5-4v96j' metadata: creationTimestamp: "2026-07-01T18:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:46:45Z" name: isvc-sklearn-graph-2-predictor-6c8478f9b5.18be3ed93f9205f8 namespace: kserve-ci-e2e-test resourceVersion: "14244" uid: 002dfd23-aef0-4de9-9c0a-4bdacb8018ba reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14230" uid: 4d0f520f-9526-4fce-af7a-cd7d26166065 kind: Event lastTimestamp: "2026-07-01T18:46:45Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-6c8478f9b5 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:46:45Z" name: isvc-sklearn-graph-2-predictor.18be3ed93adde946 namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: 59a8085a-591e-45c6-8df3-19348447bdbe reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:47:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: 333f6cd1-016b-44ae-9915-f684ed0fcbd2 kind: Event lastTimestamp: "2026-07-01T18:48:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:48:00Z" name: isvc-sklearn-graph-2-predictor.18be3edcb9442b6e namespace: kserve-ci-e2e-test resourceVersion: "14995" uid: 4fca3833-a0ab-4cf3-aa40-b56c92eb4c2a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:47:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: 333f6cd1-016b-44ae-9915-f684ed0fcbd2 kind: Event lastTimestamp: "2026-07-01T18:48:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:48:00Z" name: isvc-sklearn-graph-2-predictor.18be3edcb9c8051f namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: 91956a0e-14c9-4046-8a0c-435a2279b82d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:48:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: 333f6cd1-016b-44ae-9915-f684ed0fcbd2 kind: Event lastTimestamp: "2026-07-01T18:49:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:49:00Z" name: isvc-sklearn-graph-2-predictor.18be3eee3511dd23 namespace: kserve-ci-e2e-test resourceVersion: "15692" uid: d85db535-2531-4045-9110-2065a22966fd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:48:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: 333f6cd1-016b-44ae-9915-f684ed0fcbd2 kind: Event lastTimestamp: "2026-07-01T18:49:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:49:00Z" name: isvc-sklearn-graph-2-predictor.18be3eee358ee061 namespace: kserve-ci-e2e-test resourceVersion: "15693" uid: 067e00bb-7a3a-47e1-9954-0e0be44dc749 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T18:46:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14215" uid: b4bc6e89-5fcb-4c47-9f58-64a89d63a923 kind: Event lastTimestamp: "2026-07-01T18:48:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:48:36Z" name: isvc-sklearn-graph-2.18be3ed93ae4fde3 namespace: kserve-ci-e2e-test resourceVersion: "15500" uid: 06e9813b-79a8-4f29-a7ac-287d4c1b0862 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: b4bc6e89-5fcb-4c47-9f58-64a89d63a923 kind: Event lastTimestamp: "2026-07-01T18:48:36Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-01T18:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:48:36Z" name: isvc-sklearn-graph-2.18be3ef331559c88 namespace: kserve-ci-e2e-test resourceVersion: "15498" uid: 7c9b16cb-c9f1-41a1-8a93-1b3454979aff reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:46:44.949483Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq namespace: kserve-ci-e2e-test resourceVersion: "14188" uid: 95987d09-5a5b-46c7-a07a-f3312f69bade kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-6hxpq to ip-10-0-134-114.ec2.internal metadata: creationTimestamp: "2026-07-01T18:46:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:46:44Z" name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq.18be3ed92bd4ede2 namespace: kserve-ci-e2e-test resourceVersion: "14195" uid: 24df9e19-4143-49be-89f0-86d26160b59a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:45Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq namespace: kserve-ci-e2e-test resourceVersion: "14200" uid: 95987d09-5a5b-46c7-a07a-f3312f69bade kind: Event lastTimestamp: "2026-07-01T18:46:45Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:46:45Z" name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq.18be3ed94572bfad namespace: kserve-ci-e2e-test resourceVersion: "14256" uid: be252543-040b-4186-a100-751531931fe5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 95987d09-5a5b-46c7-a07a-f3312f69bade kind: Event lastTimestamp: "2026-07-01T18:46:45Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5df3316e2bbb1c8de0277d9a0c433b6bf041a14baf6838a11e4f2bdc5ed51f25" already present on machine metadata: creationTimestamp: "2026-07-01T18:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:46:45Z" name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq.18be3ed9474966cc namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: d27db89a-1cab-4b2e-989f-a8af392fe76f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 95987d09-5a5b-46c7-a07a-f3312f69bade kind: Event lastTimestamp: "2026-07-01T18:46:45Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T18:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:46:45Z" name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq.18be3ed949f6c4e5 namespace: kserve-ci-e2e-test resourceVersion: "14265" uid: 5d652bc9-7a31-4296-bc6a-bffdc0e4dd35 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 95987d09-5a5b-46c7-a07a-f3312f69bade kind: Event lastTimestamp: "2026-07-01T18:46:45Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T18:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:46:45Z" name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq.18be3ed94a6bdb94 namespace: kserve-ci-e2e-test resourceVersion: "14266" uid: d12a8e36-01b8-4522-93dc-3fb99ad2c581 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 95987d09-5a5b-46c7-a07a-f3312f69bade kind: Event lastTimestamp: "2026-07-01T18:46:49Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-01T18:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:46:49Z" name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq.18be3eda33f0fbf6 namespace: kserve-ci-e2e-test resourceVersion: "14338" uid: 2e3459d4-3104-4338-8513-7c232dbbb4b1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 95987d09-5a5b-46c7-a07a-f3312f69bade kind: Event lastTimestamp: "2026-07-01T18:47:17Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 28.477s (28.477s including waiting). Image size: 1305293052 bytes.' metadata: creationTimestamp: "2026-07-01T18:47:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:17Z" name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq.18be3ee0d5575d90 namespace: kserve-ci-e2e-test resourceVersion: "14605" uid: 2932a676-1224-4e39-9459-a6f2c0d82ace reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 95987d09-5a5b-46c7-a07a-f3312f69bade kind: Event lastTimestamp: "2026-07-01T18:47:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:47:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:17Z" name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq.18be3ee0db96312c namespace: kserve-ci-e2e-test resourceVersion: "14606" uid: 7957b13c-f109-42ff-9a97-0f391901e3d5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 95987d09-5a5b-46c7-a07a-f3312f69bade kind: Event lastTimestamp: "2026-07-01T18:47:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:47:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:17Z" name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq.18be3ee0dd001092 namespace: kserve-ci-e2e-test resourceVersion: "14607" uid: 9b46a4af-37ce-45e5-8d27-ca966a7c7fae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 95987d09-5a5b-46c7-a07a-f3312f69bade kind: Event lastTimestamp: "2026-07-01T18:47:17Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T18:47:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:17Z" name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq.18be3ee0dd102388 namespace: kserve-ci-e2e-test resourceVersion: "14608" uid: b6ffe475-24ee-4f95-8c77-8af7cebe028d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 95987d09-5a5b-46c7-a07a-f3312f69bade kind: Event lastTimestamp: "2026-07-01T18:47:20Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.224s (2.224s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T18:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:20Z" name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq.18be3ee161a77590 namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: f993e6fe-d15c-4aef-86e7-bdb4a5a6cff1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 95987d09-5a5b-46c7-a07a-f3312f69bade kind: Event lastTimestamp: "2026-07-01T18:47:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:20Z" name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq.18be3ee1653478ae namespace: kserve-ci-e2e-test resourceVersion: "14634" uid: 98ce9f03-5e86-4f7d-834c-452a0b5b9cf8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 95987d09-5a5b-46c7-a07a-f3312f69bade kind: Event lastTimestamp: "2026-07-01T18:47:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:20Z" name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq.18be3ee165b6d4ae namespace: kserve-ci-e2e-test resourceVersion: "14636" uid: 99694b6d-b353-4fb3-9dad-bb1fdc1a6df3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T18:47:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 95987d09-5a5b-46c7-a07a-f3312f69bade kind: Event lastTimestamp: "2026-07-01T18:49:16Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:47:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:16Z" name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq.18be3ee178b6d52a namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 6af86c8c-2c2c-4bf3-84ee-7cde62ddcb15 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 95987d09-5a5b-46c7-a07a-f3312f69bade kind: Event lastTimestamp: "2026-07-01T18:49:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:14Z" name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq.18be3efc13bfa31b namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: c23822c4-280e-4d4c-ad43-5362bbf6a310 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 95987d09-5a5b-46c7-a07a-f3312f69bade kind: Event lastTimestamp: "2026-07-01T18:49:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:14Z" name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq.18be3efc13c045f6 namespace: kserve-ci-e2e-test resourceVersion: "15977" uid: c29acb64-8d75-43f2-9940-72da239b7577 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 95987d09-5a5b-46c7-a07a-f3312f69bade kind: Event lastTimestamp: "2026-07-01T18:49:16Z" message: 'Readiness probe failed: Get "https://10.132.0.25:8643/healthz": dial tcp 10.132.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:16Z" name: isvc-xgboost-graph-predictor-555fc69d56-6hxpq.18be3efc7b1f8e8b namespace: kserve-ci-e2e-test resourceVersion: "16028" uid: 0867eca1-0ac9-420a-be11-f98187da2410 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-114.ec2.internal source: component: kubelet host: ip-10-0-134-114.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14179" uid: 5c1c56b4-c7a9-45c3-9197-fbdb00478c32 kind: Event lastTimestamp: "2026-07-01T18:46:44Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-6hxpq' metadata: creationTimestamp: "2026-07-01T18:46:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:46:44Z" name: isvc-xgboost-graph-predictor-555fc69d56.18be3ed92ad1ad81 namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: d2aebbc8-25dd-41fe-a652-31b7ffc8db11 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14177" uid: 6878c396-cdac-47fd-9507-15e81918f4b5 kind: Event lastTimestamp: "2026-07-01T18:46:44Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:46:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:46:44Z" name: isvc-xgboost-graph-predictor.18be3ed9257f3f13 namespace: kserve-ci-e2e-test resourceVersion: "14180" uid: a3f15a0d-afaf-45e6-90da-d750371cf2ed reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:46:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14178" uid: c85908f6-06f4-4262-80d9-bc2e8dadc388 kind: Event lastTimestamp: "2026-07-01T18:47:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:47:44Z" name: isvc-xgboost-graph-predictor.18be3edca42c7e4b namespace: kserve-ci-e2e-test resourceVersion: "14846" uid: 22947cb4-be34-4a46-b847-454c2fcb9916 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:46:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14178" uid: c85908f6-06f4-4262-80d9-bc2e8dadc388 kind: Event lastTimestamp: "2026-07-01T18:47:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:47:44Z" name: isvc-xgboost-graph-predictor.18be3edca4ca55ad namespace: kserve-ci-e2e-test resourceVersion: "14847" uid: 77dd82cb-f5d6-49dd-a3a7-717291294662 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:47:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14434" uid: c85908f6-06f4-4262-80d9-bc2e8dadc388 kind: Event lastTimestamp: "2026-07-01T18:48:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:48:59Z" name: isvc-xgboost-graph-predictor.18be3eeaa0136184 namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: 8ec69a57-395e-4b07-8ff4-0d404fe72aa8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:47:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14434" uid: c85908f6-06f4-4262-80d9-bc2e8dadc388 kind: Event lastTimestamp: "2026-07-01T18:48:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:48:59Z" name: isvc-xgboost-graph-predictor.18be3eeaa0a01534 namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: 3e3c91a5-e2eb-4025-8e98-15d24a131573 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T18:46:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14161" uid: 57e6825c-e212-487d-999c-b2b2f45a376e kind: Event lastTimestamp: "2026-07-01T18:48:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:46:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:48:26Z" name: isvc-xgboost-graph.18be3ed925cd1e32 namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: b924e4e2-006e-41a2-9a47-dfdc7a4558a7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: 57e6825c-e212-487d-999c-b2b2f45a376e kind: Event lastTimestamp: "2026-07-01T18:48:26Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-01T18:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:48:26Z" name: isvc-xgboost-graph.18be3ef0ddccd0be namespace: kserve-ci-e2e-test resourceVersion: "15401" uid: aa87a4d8-67b8-46e4-b7ba-a82362a811c5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:49:04.405594Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7565859b98-gvnv8 namespace: kserve-ci-e2e-test resourceVersion: "15740" uid: 0371b3ce-3a11-45f5-b918-a9363c53fc3c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-7565859b98-gvnv8 to ip-10-0-139-178.ec2.internal metadata: creationTimestamp: "2026-07-01T18:49:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:49:04Z" name: model-chainer-7565859b98-gvnv8.18be3ef9a410515a namespace: kserve-ci-e2e-test resourceVersion: "15745" uid: 789b9c0d-e18d-4a97-9a4d-b5601190fe9e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:04Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7565859b98-gvnv8 namespace: kserve-ci-e2e-test resourceVersion: "15744" uid: 0371b3ce-3a11-45f5-b918-a9363c53fc3c kind: Event lastTimestamp: "2026-07-01T18:49:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:49:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:04Z" name: model-chainer-7565859b98-gvnv8.18be3ef9b0ddc726 namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: 1b425e3c-f452-462f-8485-f645bcf391f7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:05Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7565859b98-gvnv8 namespace: kserve-ci-e2e-test resourceVersion: "15749" uid: 0371b3ce-3a11-45f5-b918-a9363c53fc3c kind: Event lastTimestamp: "2026-07-01T18:49:05Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:49:05Z" name: model-chainer-7565859b98-gvnv8.18be3ef9e14e2610 namespace: kserve-ci-e2e-test resourceVersion: "15765" uid: e2902998-885a-4ecd-ac51-12e2f45a164a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7565859b98-gvnv8 namespace: kserve-ci-e2e-test resourceVersion: "15744" uid: 0371b3ce-3a11-45f5-b918-a9363c53fc3c kind: Event lastTimestamp: "2026-07-01T18:49:05Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a589cb7fd67a31c767ba2472b47f816cadffddbd6c2e2c8ddfd876f9a79d8448" already present on machine metadata: creationTimestamp: "2026-07-01T18:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:05Z" name: model-chainer-7565859b98-gvnv8.18be3ef9e2b48846 namespace: kserve-ci-e2e-test resourceVersion: "15767" uid: 9c265484-df6c-4b82-9caf-7c4a79d34975 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7565859b98-gvnv8 namespace: kserve-ci-e2e-test resourceVersion: "15744" uid: 0371b3ce-3a11-45f5-b918-a9363c53fc3c kind: Event lastTimestamp: "2026-07-01T18:49:05Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-01T18:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:05Z" name: model-chainer-7565859b98-gvnv8.18be3ef9e57973a5 namespace: kserve-ci-e2e-test resourceVersion: "15768" uid: 9d1bc4e1-9ba5-45d5-91ed-67f566a9505b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7565859b98-gvnv8 namespace: kserve-ci-e2e-test resourceVersion: "15744" uid: 0371b3ce-3a11-45f5-b918-a9363c53fc3c kind: Event lastTimestamp: "2026-07-01T18:49:05Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-01T18:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:05Z" name: model-chainer-7565859b98-gvnv8.18be3ef9e5da9388 namespace: kserve-ci-e2e-test resourceVersion: "15769" uid: 10c3f5c1-6aed-4723-a5bd-3a02296bc0cd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7565859b98-gvnv8 namespace: kserve-ci-e2e-test resourceVersion: "15744" uid: 0371b3ce-3a11-45f5-b918-a9363c53fc3c kind: Event lastTimestamp: "2026-07-01T18:49:14Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-01T18:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:14Z" name: model-chainer-7565859b98-gvnv8.18be3efbfe6a41ff namespace: kserve-ci-e2e-test resourceVersion: "15871" uid: 7bf846fc-3244-418d-8bea-4642fdaff49a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7565859b98-gvnv8 namespace: kserve-ci-e2e-test resourceVersion: "15744" uid: 0371b3ce-3a11-45f5-b918-a9363c53fc3c kind: Event lastTimestamp: "2026-07-01T18:49:41Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T18:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:41Z" name: model-chainer-7565859b98-gvnv8.18be3efc8afcecf5 namespace: kserve-ci-e2e-test resourceVersion: "16283" uid: e37d0220-b4ee-4355-9888-f73ffed48bcc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-7565859b98 namespace: kserve-ci-e2e-test resourceVersion: "15737" uid: c89238e4-5cfe-49b6-8e9a-a357064f296b kind: Event lastTimestamp: "2026-07-01T18:49:04Z" message: 'Created pod: model-chainer-7565859b98-gvnv8' metadata: creationTimestamp: "2026-07-01T18:49:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:49:04Z" name: model-chainer-7565859b98.18be3ef9a3933c2f namespace: kserve-ci-e2e-test resourceVersion: "15742" uid: 9e04b115-ecd2-4975-9c54-3087ae02dfd7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: d4a7173d-08c4-4d2a-ad6f-caebd6e1596e kind: Event lastTimestamp: "2026-07-01T18:49:04Z" message: Scaled up replica set model-chainer-7565859b98 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:49:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:49:04Z" name: model-chainer.18be3ef9a2a3e810 namespace: kserve-ci-e2e-test resourceVersion: "15738" uid: 371b9d48-0d11-4400-8d22-d8aaddf2a4b4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15751" uid: 8b06bbb8-f387-47aa-b191-c137b1682f82 kind: Event lastTimestamp: "2026-07-01T18:49:04Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-01T18:49:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:49:04Z" name: model-chainer.18be3ef9a6eeaee4 namespace: kserve-ci-e2e-test resourceVersion: "15753" uid: a34f12f1-e32f-435e-b389-f12113885694 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:59:39.509114Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-550f3-686769c68-t5pr2 namespace: kserve-ci-e2e-test resourceVersion: "21125" uid: 9c8d03ab-9bb3-4ac6-8794-aa4b47db3fb4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-550f3-686769c68-t5pr2 to ip-10-0-139-178.ec2.internal metadata: creationTimestamp: "2026-07-01T18:59:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:59:39Z" name: sequence-graph-550f3-686769c68-t5pr2.18be3f8d832ee27f namespace: kserve-ci-e2e-test resourceVersion: "21130" uid: 6cbb2744-b997-447f-9d60-d18d7009d802 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:39Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-550f3-686769c68-t5pr2 namespace: kserve-ci-e2e-test resourceVersion: "21133" uid: 9c8d03ab-9bb3-4ac6-8794-aa4b47db3fb4 kind: Event lastTimestamp: "2026-07-01T18:59:39Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:59:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:59:39Z" name: sequence-graph-550f3-686769c68-t5pr2.18be3f8d9d0949f9 namespace: kserve-ci-e2e-test resourceVersion: "21145" uid: a5614148-1989-421e-9114-42503b4fffb2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-550f3} kind: Pod name: sequence-graph-550f3-686769c68-t5pr2 namespace: kserve-ci-e2e-test resourceVersion: "21128" uid: 9c8d03ab-9bb3-4ac6-8794-aa4b47db3fb4 kind: Event lastTimestamp: "2026-07-01T18:59:39Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a589cb7fd67a31c767ba2472b47f816cadffddbd6c2e2c8ddfd876f9a79d8448" already present on machine metadata: creationTimestamp: "2026-07-01T18:59:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:39Z" name: sequence-graph-550f3-686769c68-t5pr2.18be3f8d9e661fa8 namespace: kserve-ci-e2e-test resourceVersion: "21147" uid: 924f7516-9b59-4679-957a-d407181dca4f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-550f3} kind: Pod name: sequence-graph-550f3-686769c68-t5pr2 namespace: kserve-ci-e2e-test resourceVersion: "21128" uid: 9c8d03ab-9bb3-4ac6-8794-aa4b47db3fb4 kind: Event lastTimestamp: "2026-07-01T18:59:40Z" message: 'Created container: sequence-graph-550f3' metadata: creationTimestamp: "2026-07-01T18:59:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:40Z" name: sequence-graph-550f3-686769c68-t5pr2.18be3f8da1483bb7 namespace: kserve-ci-e2e-test resourceVersion: "21148" uid: e7324735-6afe-410d-949e-6cebb6a7ea33 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-550f3} kind: Pod name: sequence-graph-550f3-686769c68-t5pr2 namespace: kserve-ci-e2e-test resourceVersion: "21128" uid: 9c8d03ab-9bb3-4ac6-8794-aa4b47db3fb4 kind: Event lastTimestamp: "2026-07-01T18:59:40Z" message: Started container sequence-graph-550f3 metadata: creationTimestamp: "2026-07-01T18:59:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:40Z" name: sequence-graph-550f3-686769c68-t5pr2.18be3f8da21195cb namespace: kserve-ci-e2e-test resourceVersion: "21149" uid: b8275322-1249-46da-878b-9639e64549ca reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-550f3} kind: Pod name: sequence-graph-550f3-686769c68-t5pr2 namespace: kserve-ci-e2e-test resourceVersion: "21128" uid: 9c8d03ab-9bb3-4ac6-8794-aa4b47db3fb4 kind: Event lastTimestamp: "2026-07-01T18:59:53Z" message: Stopping container sequence-graph-550f3 metadata: creationTimestamp: "2026-07-01T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:53Z" name: sequence-graph-550f3-686769c68-t5pr2.18be3f90cd375547 namespace: kserve-ci-e2e-test resourceVersion: "21342" uid: 7d77aa8c-ad77-4eb5-b635-68bab23bbc88 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:59:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-550f3} kind: Pod name: sequence-graph-550f3-686769c68-t5pr2 namespace: kserve-ci-e2e-test resourceVersion: "21128" uid: 9c8d03ab-9bb3-4ac6-8794-aa4b47db3fb4 kind: Event lastTimestamp: "2026-07-01T19:00:21Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T18:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:00:21Z" name: sequence-graph-550f3-686769c68-t5pr2.18be3f91817389f9 namespace: kserve-ci-e2e-test resourceVersion: "21721" uid: 07648bc4-8748-48f6-b8cd-6dd4ad58f7c7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-550f3-686769c68 namespace: kserve-ci-e2e-test resourceVersion: "21122" uid: cef12ee3-4940-4c4e-923a-04a8de6c5c11 kind: Event lastTimestamp: "2026-07-01T18:59:39Z" message: 'Created pod: sequence-graph-550f3-686769c68-t5pr2' metadata: creationTimestamp: "2026-07-01T18:59:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:59:39Z" name: sequence-graph-550f3-686769c68.18be3f8d82bde264 namespace: kserve-ci-e2e-test resourceVersion: "21127" uid: f6089639-614f-49bb-9482-f6082d8c1119 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-550f3 namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: 40e4a78c-a1bd-4443-8145-87e46fcb6cf9 kind: Event lastTimestamp: "2026-07-01T18:59:39Z" message: Scaled up replica set sequence-graph-550f3-686769c68 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:59:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:59:39Z" name: sequence-graph-550f3.18be3f8d81c9efd1 namespace: kserve-ci-e2e-test resourceVersion: "21124" uid: 8b200a20-449d-42ed-9668-b3ddb7c56d6e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-550f3 namespace: kserve-ci-e2e-test resourceVersion: "21136" uid: 5b856293-4415-47d5-ab7b-103ac5d3a6ed kind: Event lastTimestamp: "2026-07-01T18:59:39Z" message: InferenceGraph [sequence-graph-550f3] is Ready metadata: creationTimestamp: "2026-07-01T18:59:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:59:39Z" name: sequence-graph-550f3.18be3f8d87b7e417 namespace: kserve-ci-e2e-test resourceVersion: "21138" uid: 4f738683-8ae4-490f-99cb-bdbb17d61495 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:00:53.886747Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c3262-557db7f5b8-g279b namespace: kserve-ci-e2e-test resourceVersion: "21991" uid: 12a0c79a-b5f5-43ef-a3db-b82cca330ffa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-c3262-557db7f5b8-g279b to ip-10-0-139-178.ec2.internal metadata: creationTimestamp: "2026-07-01T19:00:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:00:53Z" name: sequence-graph-c3262-557db7f5b8-g279b.18be3f9ed46f7850 namespace: kserve-ci-e2e-test resourceVersion: "21996" uid: 52715aa1-1c27-46d8-9e8e-7e4ac1a388fa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:00:54Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c3262-557db7f5b8-g279b namespace: kserve-ci-e2e-test resourceVersion: "22000" uid: 12a0c79a-b5f5-43ef-a3db-b82cca330ffa kind: Event lastTimestamp: "2026-07-01T19:00:54Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:00:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:00:54Z" name: sequence-graph-c3262-557db7f5b8-g279b.18be3f9eee1ae274 namespace: kserve-ci-e2e-test resourceVersion: "22011" uid: ef7424c5-78aa-4a96-907a-0b222a8b8584 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:00:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c3262} kind: Pod name: sequence-graph-c3262-557db7f5b8-g279b namespace: kserve-ci-e2e-test resourceVersion: "21993" uid: 12a0c79a-b5f5-43ef-a3db-b82cca330ffa kind: Event lastTimestamp: "2026-07-01T19:00:54Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a589cb7fd67a31c767ba2472b47f816cadffddbd6c2e2c8ddfd876f9a79d8448" already present on machine metadata: creationTimestamp: "2026-07-01T19:00:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:00:54Z" name: sequence-graph-c3262-557db7f5b8-g279b.18be3f9eefd0cf1e namespace: kserve-ci-e2e-test resourceVersion: "22013" uid: 211ffd4c-fee9-437a-8148-9cc6d87c21fb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:00:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c3262} kind: Pod name: sequence-graph-c3262-557db7f5b8-g279b namespace: kserve-ci-e2e-test resourceVersion: "21993" uid: 12a0c79a-b5f5-43ef-a3db-b82cca330ffa kind: Event lastTimestamp: "2026-07-01T19:00:54Z" message: 'Created container: sequence-graph-c3262' metadata: creationTimestamp: "2026-07-01T19:00:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:00:54Z" name: sequence-graph-c3262-557db7f5b8-g279b.18be3f9ef2db192a namespace: kserve-ci-e2e-test resourceVersion: "22014" uid: 53cf19a0-7e80-44b8-b5ae-0daa8b1bab98 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:00:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c3262} kind: Pod name: sequence-graph-c3262-557db7f5b8-g279b namespace: kserve-ci-e2e-test resourceVersion: "21993" uid: 12a0c79a-b5f5-43ef-a3db-b82cca330ffa kind: Event lastTimestamp: "2026-07-01T19:00:54Z" message: Started container sequence-graph-c3262 metadata: creationTimestamp: "2026-07-01T19:00:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:00:54Z" name: sequence-graph-c3262-557db7f5b8-g279b.18be3f9ef359fb78 namespace: kserve-ci-e2e-test resourceVersion: "22015" uid: 52ed29b1-b3d7-41f9-bc24-f422fb94c1ac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c3262} kind: Pod name: sequence-graph-c3262-557db7f5b8-g279b namespace: kserve-ci-e2e-test resourceVersion: "21993" uid: 12a0c79a-b5f5-43ef-a3db-b82cca330ffa kind: Event lastTimestamp: "2026-07-01T19:09:08Z" message: Stopping container sequence-graph-c3262 metadata: creationTimestamp: "2026-07-01T19:09:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:08Z" name: sequence-graph-c3262-557db7f5b8-g279b.18be4011fcae7775 namespace: kserve-ci-e2e-test resourceVersion: "25929" uid: 798691d5-a1b3-49d1-97f3-727913b4c2dc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c3262} kind: Pod name: sequence-graph-c3262-557db7f5b8-g279b namespace: kserve-ci-e2e-test resourceVersion: "21993" uid: 12a0c79a-b5f5-43ef-a3db-b82cca330ffa kind: Event lastTimestamp: "2026-07-01T19:09:35Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:35Z" name: sequence-graph-c3262-557db7f5b8-g279b.18be40129209c811 namespace: kserve-ci-e2e-test resourceVersion: "26435" uid: 14e3f87b-d2f7-4b11-b13d-29e60edb9313 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:00:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-c3262-557db7f5b8 namespace: kserve-ci-e2e-test resourceVersion: "21988" uid: 7f119618-9b45-4e5c-adf9-9f482c0ef095 kind: Event lastTimestamp: "2026-07-01T19:00:53Z" message: 'Created pod: sequence-graph-c3262-557db7f5b8-g279b' metadata: creationTimestamp: "2026-07-01T19:00:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:00:53Z" name: sequence-graph-c3262-557db7f5b8.18be3f9ed3cb310f namespace: kserve-ci-e2e-test resourceVersion: "21992" uid: 44d3d9dc-ea3b-42f9-a892-6d144cc7ff59 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:00:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-c3262 namespace: kserve-ci-e2e-test resourceVersion: "21986" uid: 0efd43c7-2fd6-45e6-badb-297e4a6cc76c kind: Event lastTimestamp: "2026-07-01T19:00:53Z" message: Scaled up replica set sequence-graph-c3262-557db7f5b8 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:00:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:00:53Z" name: sequence-graph-c3262.18be3f9ed2cdb657 namespace: kserve-ci-e2e-test resourceVersion: "21989" uid: 7ff2fb12-5515-47f9-b643-a448912b5a4a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:00:53Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-c3262 namespace: kserve-ci-e2e-test resourceVersion: "22004" uid: ab641f91-c5ba-44d8-ae0f-b1b25501914c kind: Event lastTimestamp: "2026-07-01T19:00:53Z" message: InferenceGraph [sequence-graph-c3262] is Ready metadata: creationTimestamp: "2026-07-01T19:00:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:00:53Z" name: sequence-graph-c3262.18be3f9ed9f627c0 namespace: kserve-ci-e2e-test resourceVersion: "22005" uid: 6f5a3132-e08b-4b1b-b998-4049367f3fed reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:01:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c3262 namespace: kserve-ci-e2e-test resourceVersion: "21987" uid: 710b10f2-6d50-400b-8261-afb904abc4c4 kind: Event lastTimestamp: "2026-07-01T19:01:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:01:08Z" name: sequence-graph-c3262.18be3fa251567f2b namespace: kserve-ci-e2e-test resourceVersion: "22122" uid: 5f29096b-7dd7-4a04-b5ec-82930bfe6a9d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:01:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c3262 namespace: kserve-ci-e2e-test resourceVersion: "21987" uid: 710b10f2-6d50-400b-8261-afb904abc4c4 kind: Event lastTimestamp: "2026-07-01T19:01:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:01:08Z" name: sequence-graph-c3262.18be3fa251e385b6 namespace: kserve-ci-e2e-test resourceVersion: "22124" uid: 3d01e7db-161f-468f-84c4-a69e57ef46c4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:01:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c3262 namespace: kserve-ci-e2e-test resourceVersion: "22123" uid: 710b10f2-6d50-400b-8261-afb904abc4c4 kind: Event lastTimestamp: "2026-07-01T19:01:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:01:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:01:23Z" name: sequence-graph-c3262.18be3fa5d0dedbb2 namespace: kserve-ci-e2e-test resourceVersion: "22231" uid: 4a319413-d888-426e-852d-01dc591fc6ca reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:01:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c3262 namespace: kserve-ci-e2e-test resourceVersion: "22123" uid: 710b10f2-6d50-400b-8261-afb904abc4c4 kind: Event lastTimestamp: "2026-07-01T19:01:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:01:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:01:23Z" name: sequence-graph-c3262.18be3fa5d1636d74 namespace: kserve-ci-e2e-test resourceVersion: "22233" uid: 98d9383f-0fdc-4679-9f78-63b7f5f974b8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:50:14.699685Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-eb211-648cf9b67b-h6dpc namespace: kserve-ci-e2e-test resourceVersion: "16555" uid: ce6acf93-bdda-4f40-9e82-3ecafb59c829 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-eb211-648cf9b67b-h6dpc to ip-10-0-139-178.ec2.internal metadata: creationTimestamp: "2026-07-01T18:50:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:50:14Z" name: sequence-graph-eb211-648cf9b67b-h6dpc.18be3f0a01eaef28 namespace: kserve-ci-e2e-test resourceVersion: "16561" uid: 2d0a1b21-68e3-4f90-af64-6019fc350980 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:50:15Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-eb211-648cf9b67b-h6dpc namespace: kserve-ci-e2e-test resourceVersion: "16564" uid: ce6acf93-bdda-4f40-9e82-3ecafb59c829 kind: Event lastTimestamp: "2026-07-01T18:50:15Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:50:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:50:15Z" name: sequence-graph-eb211-648cf9b67b-h6dpc.18be3f0a1b18368c namespace: kserve-ci-e2e-test resourceVersion: "16579" uid: bf0621f8-4521-4780-b1a2-b11a16edd2a0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:50:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-eb211} kind: Pod name: sequence-graph-eb211-648cf9b67b-h6dpc namespace: kserve-ci-e2e-test resourceVersion: "16558" uid: ce6acf93-bdda-4f40-9e82-3ecafb59c829 kind: Event lastTimestamp: "2026-07-01T18:50:15Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a589cb7fd67a31c767ba2472b47f816cadffddbd6c2e2c8ddfd876f9a79d8448" already present on machine metadata: creationTimestamp: "2026-07-01T18:50:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:50:15Z" name: sequence-graph-eb211-648cf9b67b-h6dpc.18be3f0a1c847608 namespace: kserve-ci-e2e-test resourceVersion: "16581" uid: e12e423f-6de2-4bce-b43d-c131e3cabb1d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:50:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-eb211} kind: Pod name: sequence-graph-eb211-648cf9b67b-h6dpc namespace: kserve-ci-e2e-test resourceVersion: "16558" uid: ce6acf93-bdda-4f40-9e82-3ecafb59c829 kind: Event lastTimestamp: "2026-07-01T18:50:15Z" message: 'Created container: sequence-graph-eb211' metadata: creationTimestamp: "2026-07-01T18:50:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:50:15Z" name: sequence-graph-eb211-648cf9b67b-h6dpc.18be3f0a1f1a46b1 namespace: kserve-ci-e2e-test resourceVersion: "16582" uid: afc5da93-7941-44b2-9c3d-e241f8e168a1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:50:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-eb211} kind: Pod name: sequence-graph-eb211-648cf9b67b-h6dpc namespace: kserve-ci-e2e-test resourceVersion: "16558" uid: ce6acf93-bdda-4f40-9e82-3ecafb59c829 kind: Event lastTimestamp: "2026-07-01T18:50:15Z" message: Started container sequence-graph-eb211 metadata: creationTimestamp: "2026-07-01T18:50:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:50:15Z" name: sequence-graph-eb211-648cf9b67b-h6dpc.18be3f0a1f7ca4dd namespace: kserve-ci-e2e-test resourceVersion: "16583" uid: 7e09ccdb-3655-4bc2-858b-eff4e79cdc99 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-eb211} kind: Pod name: sequence-graph-eb211-648cf9b67b-h6dpc namespace: kserve-ci-e2e-test resourceVersion: "16558" uid: ce6acf93-bdda-4f40-9e82-3ecafb59c829 kind: Event lastTimestamp: "2026-07-01T18:58:29Z" message: Stopping container sequence-graph-eb211 metadata: creationTimestamp: "2026-07-01T18:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:29Z" name: sequence-graph-eb211-648cf9b67b-h6dpc.18be3f7d2bab8827 namespace: kserve-ci-e2e-test resourceVersion: "20198" uid: 8ff2d35a-9a45-4db0-b616-5346b11b7d0a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:58:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-eb211} kind: Pod name: sequence-graph-eb211-648cf9b67b-h6dpc namespace: kserve-ci-e2e-test resourceVersion: "16558" uid: ce6acf93-bdda-4f40-9e82-3ecafb59c829 kind: Event lastTimestamp: "2026-07-01T18:58:57Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T18:58:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:57Z" name: sequence-graph-eb211-648cf9b67b-h6dpc.18be3f7dd63aba4e namespace: kserve-ci-e2e-test resourceVersion: "20740" uid: 0acf8836-9698-4ef6-a85e-cfac3ee77506 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:50:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-eb211-648cf9b67b namespace: kserve-ci-e2e-test resourceVersion: "16552" uid: 9973e0bb-7f6e-4739-b08c-85f72f0c9d14 kind: Event lastTimestamp: "2026-07-01T18:50:14Z" message: 'Created pod: sequence-graph-eb211-648cf9b67b-h6dpc' metadata: creationTimestamp: "2026-07-01T18:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:50:14Z" name: sequence-graph-eb211-648cf9b67b.18be3f0a0165150a namespace: kserve-ci-e2e-test resourceVersion: "16557" uid: ff41a3e9-9fdc-4346-b0df-25257ef62f30 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:50:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-eb211 namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: ca32d7b0-9fa3-4545-92ca-96f74dbffa62 kind: Event lastTimestamp: "2026-07-01T18:50:14Z" message: Scaled up replica set sequence-graph-eb211-648cf9b67b from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:50:14Z" name: sequence-graph-eb211.18be3f0a007b4905 namespace: kserve-ci-e2e-test resourceVersion: "16553" uid: b96481a8-4309-4e7a-a459-064e89fe97f1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:50:14Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-eb211 namespace: kserve-ci-e2e-test resourceVersion: "16568" uid: 5060c30a-28ea-4bb6-bb66-45f43e7975c0 kind: Event lastTimestamp: "2026-07-01T18:50:14Z" message: InferenceGraph [sequence-graph-eb211] is Ready metadata: creationTimestamp: "2026-07-01T18:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:50:14Z" name: sequence-graph-eb211.18be3f0a065fa314 namespace: kserve-ci-e2e-test resourceVersion: "16570" uid: ac68e9c1-a928-48dd-b944-42451e01dab7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:50:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-eb211 namespace: kserve-ci-e2e-test resourceVersion: "16551" uid: 24dfc7d8-dadb-4c85-8810-b63f42ef8abc kind: Event lastTimestamp: "2026-07-01T18:50:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:50:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:50:44Z" name: sequence-graph-eb211.18be3f0d7fb74ec6 namespace: kserve-ci-e2e-test resourceVersion: "16803" uid: 6cfc4226-1b73-48ab-97e0-69d6a485a4c2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:50:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-eb211 namespace: kserve-ci-e2e-test resourceVersion: "16551" uid: 24dfc7d8-dadb-4c85-8810-b63f42ef8abc kind: Event lastTimestamp: "2026-07-01T18:50:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:50:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:50:44Z" name: sequence-graph-eb211.18be3f0d7ffe8ec4 namespace: kserve-ci-e2e-test resourceVersion: "16804" uid: 9938fd8f-1c67-4a84-a2b5-92f8a788525b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:08:58.944228Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-1bf2c-997f87564-8nkrj namespace: kserve-ci-e2e-test resourceVersion: "25526" uid: 79262580-0764-4917-876d-6523169babe7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-1bf2c-997f87564-8nkrj to ip-10-0-139-178.ec2.internal metadata: creationTimestamp: "2026-07-01T19:08:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:08:58Z" name: splitter-graph-1bf2c-997f87564-8nkrj.18be400fc41d46b7 namespace: kserve-ci-e2e-test resourceVersion: "25533" uid: 0d1085be-538a-45cd-8117-6d146144d8d2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:08:59Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-1bf2c-997f87564-8nkrj namespace: kserve-ci-e2e-test resourceVersion: "25538" uid: 79262580-0764-4917-876d-6523169babe7 kind: Event lastTimestamp: "2026-07-01T19:08:59Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:08:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:08:59Z" name: splitter-graph-1bf2c-997f87564-8nkrj.18be400fddb8df8c namespace: kserve-ci-e2e-test resourceVersion: "25555" uid: bb08ebd0-0fe7-47c5-b329-73ad4767c063 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:08:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-1bf2c} kind: Pod name: splitter-graph-1bf2c-997f87564-8nkrj namespace: kserve-ci-e2e-test resourceVersion: "25531" uid: 79262580-0764-4917-876d-6523169babe7 kind: Event lastTimestamp: "2026-07-01T19:08:59Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a589cb7fd67a31c767ba2472b47f816cadffddbd6c2e2c8ddfd876f9a79d8448" already present on machine metadata: creationTimestamp: "2026-07-01T19:08:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:08:59Z" name: splitter-graph-1bf2c-997f87564-8nkrj.18be400fdf30f2a1 namespace: kserve-ci-e2e-test resourceVersion: "25557" uid: 37a70b40-42f7-4ef4-9ed8-04dd46e0cec2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:08:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-1bf2c} kind: Pod name: splitter-graph-1bf2c-997f87564-8nkrj namespace: kserve-ci-e2e-test resourceVersion: "25531" uid: 79262580-0764-4917-876d-6523169babe7 kind: Event lastTimestamp: "2026-07-01T19:08:59Z" message: 'Created container: splitter-graph-1bf2c' metadata: creationTimestamp: "2026-07-01T19:08:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:08:59Z" name: splitter-graph-1bf2c-997f87564-8nkrj.18be400fe21eb25a namespace: kserve-ci-e2e-test resourceVersion: "25559" uid: 7171d216-0dfd-4be7-9709-53de84698958 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:08:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-1bf2c} kind: Pod name: splitter-graph-1bf2c-997f87564-8nkrj namespace: kserve-ci-e2e-test resourceVersion: "25531" uid: 79262580-0764-4917-876d-6523169babe7 kind: Event lastTimestamp: "2026-07-01T19:08:59Z" message: Started container splitter-graph-1bf2c metadata: creationTimestamp: "2026-07-01T19:08:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:08:59Z" name: splitter-graph-1bf2c-997f87564-8nkrj.18be400fe2972aac namespace: kserve-ci-e2e-test resourceVersion: "25560" uid: 30276cf7-e920-411e-8079-fc290b6f906d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-1bf2c} kind: Pod name: splitter-graph-1bf2c-997f87564-8nkrj namespace: kserve-ci-e2e-test resourceVersion: "25531" uid: 79262580-0764-4917-876d-6523169babe7 kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: Stopping container splitter-graph-1bf2c metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:09Z" name: splitter-graph-1bf2c-997f87564-8nkrj.18be40121be28f45 namespace: kserve-ci-e2e-test resourceVersion: "26042" uid: e9a7c77d-1ba0-4d8e-ae9b-b814354c9eff reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:09:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-1bf2c} kind: Pod name: splitter-graph-1bf2c-997f87564-8nkrj namespace: kserve-ci-e2e-test resourceVersion: "25531" uid: 79262580-0764-4917-876d-6523169babe7 kind: Event lastTimestamp: "2026-07-01T19:09:36Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T19:09:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:36Z" name: splitter-graph-1bf2c-997f87564-8nkrj.18be40129b794fa7 namespace: kserve-ci-e2e-test resourceVersion: "26437" uid: c146a782-07cc-4020-8c52-fbb7d48960f4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:08:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-1bf2c-997f87564 namespace: kserve-ci-e2e-test resourceVersion: "25522" uid: 0ba9e01b-de4e-4167-acfc-0be87450832d kind: Event lastTimestamp: "2026-07-01T19:08:58Z" message: 'Created pod: splitter-graph-1bf2c-997f87564-8nkrj' metadata: creationTimestamp: "2026-07-01T19:08:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:08:58Z" name: splitter-graph-1bf2c-997f87564.18be400fc3955412 namespace: kserve-ci-e2e-test resourceVersion: "25529" uid: 925ff5fc-187c-4f57-bf79-f025c61fcdc1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:08:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-1bf2c namespace: kserve-ci-e2e-test resourceVersion: "25520" uid: 4f222806-4bfd-4f8d-8d07-dda93408ed10 kind: Event lastTimestamp: "2026-07-01T19:08:58Z" message: Scaled up replica set splitter-graph-1bf2c-997f87564 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:08:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:08:58Z" name: splitter-graph-1bf2c.18be400fc2a2ba7b namespace: kserve-ci-e2e-test resourceVersion: "25524" uid: 9cf0e850-5394-4e8f-9f38-c5550cac8e0d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:08:59Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-1bf2c namespace: kserve-ci-e2e-test resourceVersion: "25541" uid: d14bcd82-0ea1-42ef-b08d-d84ab3cb6169 kind: Event lastTimestamp: "2026-07-01T19:08:59Z" message: InferenceGraph [splitter-graph-1bf2c] is Ready metadata: creationTimestamp: "2026-07-01T19:08:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:08:59Z" name: splitter-graph-1bf2c.18be400fc8f8b90b namespace: kserve-ci-e2e-test resourceVersion: "25544" uid: 79fa8985-3616-41d7-aba1-76c3771996aa reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:10:09.233557Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-4f643-59d4664c7d-tk7t6 namespace: kserve-ci-e2e-test resourceVersion: "26752" uid: aa90ba40-136c-45ea-88c2-b1b84275bae4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-4f643-59d4664c7d-tk7t6 to ip-10-0-139-178.ec2.internal metadata: creationTimestamp: "2026-07-01T19:10:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:10:09Z" name: splitter-graph-4f643-59d4664c7d-tk7t6.18be402021af50d9 namespace: kserve-ci-e2e-test resourceVersion: "26758" uid: 7b7f8145-0689-4a60-af7f-f28c9cd651b6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:10:09Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-4f643-59d4664c7d-tk7t6 namespace: kserve-ci-e2e-test resourceVersion: "26755" uid: aa90ba40-136c-45ea-88c2-b1b84275bae4 kind: Event lastTimestamp: "2026-07-01T19:10:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-4f643-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:10:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:10:09Z" name: splitter-graph-4f643-59d4664c7d-tk7t6.18be40202925cde4 namespace: kserve-ci-e2e-test resourceVersion: "26771" uid: 799e3802-fcba-421f-91a4-f4ee3fdc4949 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:10:10Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-4f643-59d4664c7d-tk7t6 namespace: kserve-ci-e2e-test resourceVersion: "26761" uid: aa90ba40-136c-45ea-88c2-b1b84275bae4 kind: Event lastTimestamp: "2026-07-01T19:10:10Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:10:10Z" name: splitter-graph-4f643-59d4664c7d-tk7t6.18be40205f001f04 namespace: kserve-ci-e2e-test resourceVersion: "26778" uid: d67fc0a4-a17e-4d50-ac7c-021fb2758f27 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:10:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4f643} kind: Pod name: splitter-graph-4f643-59d4664c7d-tk7t6 namespace: kserve-ci-e2e-test resourceVersion: "26755" uid: aa90ba40-136c-45ea-88c2-b1b84275bae4 kind: Event lastTimestamp: "2026-07-01T19:10:10Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a589cb7fd67a31c767ba2472b47f816cadffddbd6c2e2c8ddfd876f9a79d8448" already present on machine metadata: creationTimestamp: "2026-07-01T19:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:10:10Z" name: splitter-graph-4f643-59d4664c7d-tk7t6.18be402060539a26 namespace: kserve-ci-e2e-test resourceVersion: "26780" uid: 81f0cd96-4993-4479-8afa-b95ea807632d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:10:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4f643} kind: Pod name: splitter-graph-4f643-59d4664c7d-tk7t6 namespace: kserve-ci-e2e-test resourceVersion: "26755" uid: aa90ba40-136c-45ea-88c2-b1b84275bae4 kind: Event lastTimestamp: "2026-07-01T19:10:10Z" message: 'Created container: splitter-graph-4f643' metadata: creationTimestamp: "2026-07-01T19:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:10:10Z" name: splitter-graph-4f643-59d4664c7d-tk7t6.18be40206336c2b9 namespace: kserve-ci-e2e-test resourceVersion: "26781" uid: 78f249ef-fb3f-418b-954c-0cdefb60246f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:10:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4f643} kind: Pod name: splitter-graph-4f643-59d4664c7d-tk7t6 namespace: kserve-ci-e2e-test resourceVersion: "26755" uid: aa90ba40-136c-45ea-88c2-b1b84275bae4 kind: Event lastTimestamp: "2026-07-01T19:10:10Z" message: Started container splitter-graph-4f643 metadata: creationTimestamp: "2026-07-01T19:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:10:10Z" name: splitter-graph-4f643-59d4664c7d-tk7t6.18be402063a6056b namespace: kserve-ci-e2e-test resourceVersion: "26782" uid: 038c6279-c0c7-47dd-ab1e-c91a436d2e30 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:18:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4f643} kind: Pod name: splitter-graph-4f643-59d4664c7d-tk7t6 namespace: kserve-ci-e2e-test resourceVersion: "26755" uid: aa90ba40-136c-45ea-88c2-b1b84275bae4 kind: Event lastTimestamp: "2026-07-01T19:18:23Z" message: Stopping container splitter-graph-4f643 metadata: creationTimestamp: "2026-07-01T19:18:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:18:23Z" name: splitter-graph-4f643-59d4664c7d-tk7t6.18be4093506a5ec8 namespace: kserve-ci-e2e-test resourceVersion: "30259" uid: 56a2fa07-842d-4a8c-8f50-9d9797a6d1c0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:18:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4f643} kind: Pod name: splitter-graph-4f643-59d4664c7d-tk7t6 namespace: kserve-ci-e2e-test resourceVersion: "26755" uid: aa90ba40-136c-45ea-88c2-b1b84275bae4 kind: Event lastTimestamp: "2026-07-01T19:18:51Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T19:18:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:18:51Z" name: splitter-graph-4f643-59d4664c7d-tk7t6.18be4093e6bafdb1 namespace: kserve-ci-e2e-test resourceVersion: "30496" uid: 832684ab-7155-4df5-8340-167e931d52d1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:10:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-4f643-59d4664c7d namespace: kserve-ci-e2e-test resourceVersion: "26749" uid: 05ba19a3-139e-4d51-9156-f4971ca425f4 kind: Event lastTimestamp: "2026-07-01T19:10:09Z" message: 'Created pod: splitter-graph-4f643-59d4664c7d-tk7t6' metadata: creationTimestamp: "2026-07-01T19:10:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:10:09Z" name: splitter-graph-4f643-59d4664c7d.18be4020213bceb4 namespace: kserve-ci-e2e-test resourceVersion: "26754" uid: 2c65bea7-e52a-4fce-b12d-460f256f4b84 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:10:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-4f643 namespace: kserve-ci-e2e-test resourceVersion: "26747" uid: 79898fde-868d-457a-9573-dc0d00ae6f52 kind: Event lastTimestamp: "2026-07-01T19:10:09Z" message: Scaled up replica set splitter-graph-4f643-59d4664c7d from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:10:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:10:09Z" name: splitter-graph-4f643.18be402020618d10 namespace: kserve-ci-e2e-test resourceVersion: "26750" uid: 9fda6777-5737-43c1-8b23-48ec053f947a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:10:09Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-4f643 namespace: kserve-ci-e2e-test resourceVersion: "26764" uid: 3338e803-222e-4a8c-9a6f-2f1d4ee1926d kind: Event lastTimestamp: "2026-07-01T19:10:09Z" message: InferenceGraph [splitter-graph-4f643] is Ready metadata: creationTimestamp: "2026-07-01T19:10:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:10:09Z" name: splitter-graph-4f643.18be40202478ecc6 namespace: kserve-ci-e2e-test resourceVersion: "26766" uid: 1e029c88-b313-4237-bb71-46acc35d4382 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:10:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-4f643 namespace: kserve-ci-e2e-test resourceVersion: "26748" uid: 87995c0e-7b10-4324-bf5a-e3cbbd6a073b kind: Event lastTimestamp: "2026-07-01T19:10:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:10:39Z" name: splitter-graph-4f643.18be40239ee2c08a namespace: kserve-ci-e2e-test resourceVersion: "27044" uid: 66a51833-b7c2-4609-808c-fa1a746f1fda reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:10:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-4f643 namespace: kserve-ci-e2e-test resourceVersion: "26748" uid: 87995c0e-7b10-4324-bf5a-e3cbbd6a073b kind: Event lastTimestamp: "2026-07-01T19:10:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:10:39Z" name: splitter-graph-4f643.18be40239f51a05c namespace: kserve-ci-e2e-test resourceVersion: "27045" uid: 723d98cf-3e19-4da2-b030-87411886c082 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:07:58.941994Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1bf2c-predictor-56bc8b9cd9-48jqx namespace: kserve-ci-e2e-test resourceVersion: "24954" uid: 8230e7b5-b543-452a-9a46-4fd627ad3138 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1bf2c-predictor-56bc8b9cd9-48jqx to ip-10-0-129-245.ec2.internal metadata: creationTimestamp: "2026-07-01T19:07:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:07:58Z" name: success-200-isvc-1bf2c-predictor-56bc8b9cd9-48jqx.18be4001cbb3e106 namespace: kserve-ci-e2e-test resourceVersion: "24966" uid: 0d1062e9-19d0-419e-89a9-12d8646f7d96 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:07:59Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1bf2c-predictor-56bc8b9cd9-48jqx namespace: kserve-ci-e2e-test resourceVersion: "24978" uid: 8230e7b5-b543-452a-9a46-4fd627ad3138 kind: Event lastTimestamp: "2026-07-01T19:07:59Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:07:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:07:59Z" name: success-200-isvc-1bf2c-predictor-56bc8b9cd9-48jqx.18be4001e59472c7 namespace: kserve-ci-e2e-test resourceVersion: "25024" uid: 8a67177b-f3e8-471a-b5ca-8259af6178e1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:07:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1bf2c-predictor-56bc8b9cd9-48jqx namespace: kserve-ci-e2e-test resourceVersion: "24959" uid: 8230e7b5-b543-452a-9a46-4fd627ad3138 kind: Event lastTimestamp: "2026-07-01T19:07:59Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T19:07:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:07:59Z" name: success-200-isvc-1bf2c-predictor-56bc8b9cd9-48jqx.18be4001e7042cca namespace: kserve-ci-e2e-test resourceVersion: "25026" uid: 2f086c5f-7ee5-4213-b2b5-00344a6bb13f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:07:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1bf2c-predictor-56bc8b9cd9-48jqx namespace: kserve-ci-e2e-test resourceVersion: "24959" uid: 8230e7b5-b543-452a-9a46-4fd627ad3138 kind: Event lastTimestamp: "2026-07-01T19:07:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:07:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:07:59Z" name: success-200-isvc-1bf2c-predictor-56bc8b9cd9-48jqx.18be4001e9cc92a8 namespace: kserve-ci-e2e-test resourceVersion: "25027" uid: 92f3afe2-7ac2-4208-a0e7-6c95087067aa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:07:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1bf2c-predictor-56bc8b9cd9-48jqx namespace: kserve-ci-e2e-test resourceVersion: "24959" uid: 8230e7b5-b543-452a-9a46-4fd627ad3138 kind: Event lastTimestamp: "2026-07-01T19:07:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:07:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:07:59Z" name: success-200-isvc-1bf2c-predictor-56bc8b9cd9-48jqx.18be4001ea3e118f namespace: kserve-ci-e2e-test resourceVersion: "25028" uid: b84bbebd-3c1e-47c9-b67d-3b68e32bd119 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:07:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1bf2c-predictor-56bc8b9cd9-48jqx namespace: kserve-ci-e2e-test resourceVersion: "24959" uid: 8230e7b5-b543-452a-9a46-4fd627ad3138 kind: Event lastTimestamp: "2026-07-01T19:07:59Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:07:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:07:59Z" name: success-200-isvc-1bf2c-predictor-56bc8b9cd9-48jqx.18be4001ea480896 namespace: kserve-ci-e2e-test resourceVersion: "25029" uid: edb4dda3-52bd-4168-8d46-c4934da72805 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:07:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1bf2c-predictor-56bc8b9cd9-48jqx namespace: kserve-ci-e2e-test resourceVersion: "24959" uid: 8230e7b5-b543-452a-9a46-4fd627ad3138 kind: Event lastTimestamp: "2026-07-01T19:07:59Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:07:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:07:59Z" name: success-200-isvc-1bf2c-predictor-56bc8b9cd9-48jqx.18be4001ed5a035f namespace: kserve-ci-e2e-test resourceVersion: "25030" uid: ca0441b3-248a-473d-a98a-5b19d73c1061 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:07:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1bf2c-predictor-56bc8b9cd9-48jqx namespace: kserve-ci-e2e-test resourceVersion: "24959" uid: 8230e7b5-b543-452a-9a46-4fd627ad3138 kind: Event lastTimestamp: "2026-07-01T19:07:59Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:07:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:07:59Z" name: success-200-isvc-1bf2c-predictor-56bc8b9cd9-48jqx.18be4001ede7925a namespace: kserve-ci-e2e-test resourceVersion: "25031" uid: ac80f69c-f4e3-49c4-a8e5-715d9bf84e5d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:08:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1bf2c-predictor-56bc8b9cd9-48jqx namespace: kserve-ci-e2e-test resourceVersion: "24959" uid: 8230e7b5-b543-452a-9a46-4fd627ad3138 kind: Event lastTimestamp: "2026-07-01T19:08:36Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:08:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:08:36Z" name: success-200-isvc-1bf2c-predictor-56bc8b9cd9-48jqx.18be40022f13a530 namespace: kserve-ci-e2e-test resourceVersion: "25343" uid: af977ca4-b474-48d1-ba58-af6daf403657 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1bf2c-predictor-56bc8b9cd9-48jqx namespace: kserve-ci-e2e-test resourceVersion: "24959" uid: 8230e7b5-b543-452a-9a46-4fd627ad3138 kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:09Z" name: success-200-isvc-1bf2c-predictor-56bc8b9cd9-48jqx.18be4012259df141 namespace: kserve-ci-e2e-test resourceVersion: "26101" uid: b345b3e1-5f0a-4c4e-a4b4-6716f5cd5877 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1bf2c-predictor-56bc8b9cd9-48jqx namespace: kserve-ci-e2e-test resourceVersion: "24959" uid: 8230e7b5-b543-452a-9a46-4fd627ad3138 kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:09Z" name: success-200-isvc-1bf2c-predictor-56bc8b9cd9-48jqx.18be4012259e7586 namespace: kserve-ci-e2e-test resourceVersion: "26104" uid: 02ec8690-73b1-4362-a7e8-74a2032cf7d2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1bf2c-predictor-56bc8b9cd9-48jqx namespace: kserve-ci-e2e-test resourceVersion: "24959" uid: 8230e7b5-b543-452a-9a46-4fd627ad3138 kind: Event lastTimestamp: "2026-07-01T19:09:11Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:09:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:11Z" name: success-200-isvc-1bf2c-predictor-56bc8b9cd9-48jqx.18be4012b73ae325 namespace: kserve-ci-e2e-test resourceVersion: "26219" uid: c7aa1fda-5591-4ad0-8e4a-19e09281f40c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:07:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1bf2c-predictor-56bc8b9cd9 namespace: kserve-ci-e2e-test resourceVersion: "24935" uid: 8b119d6c-d7c2-4e9f-b2c1-334c5956ae94 kind: Event lastTimestamp: "2026-07-01T19:07:58Z" message: 'Created pod: success-200-isvc-1bf2c-predictor-56bc8b9cd9-48jqx' metadata: creationTimestamp: "2026-07-01T19:07:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:07:58Z" name: success-200-isvc-1bf2c-predictor-56bc8b9cd9.18be4001cb11f212 namespace: kserve-ci-e2e-test resourceVersion: "24957" uid: 09b47597-9ef5-4011-a9e9-263c4f483082 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:07:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1bf2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24933" uid: 84c0d8fd-2eb9-4f20-b834-1b140293d96a kind: Event lastTimestamp: "2026-07-01T19:07:58Z" message: Scaled up replica set success-200-isvc-1bf2c-predictor-56bc8b9cd9 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:07:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:07:58Z" name: success-200-isvc-1bf2c-predictor.18be4001c65dea5d namespace: kserve-ci-e2e-test resourceVersion: "24936" uid: b81d57c8-91ef-497c-9fb2-725064f04731 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:08:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1bf2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24934" uid: f53dbaa4-9e5f-4e6e-bf9f-61ff0d690e3f kind: Event lastTimestamp: "2026-07-01T19:08:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:08:28Z" name: success-200-isvc-1bf2c-predictor.18be400544e9b0d6 namespace: kserve-ci-e2e-test resourceVersion: "25281" uid: 7240f3be-9d72-4823-9f2e-9990d8f41da0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:08:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1bf2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "24934" uid: f53dbaa4-9e5f-4e6e-bf9f-61ff0d690e3f kind: Event lastTimestamp: "2026-07-01T19:08:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:08:28Z" name: success-200-isvc-1bf2c-predictor.18be4005454a3b69 namespace: kserve-ci-e2e-test resourceVersion: "25282" uid: d82d453d-7707-4e70-a977-44e14039621b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:08:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1bf2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25163" uid: f53dbaa4-9e5f-4e6e-bf9f-61ff0d690e3f kind: Event lastTimestamp: "2026-07-01T19:08:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:08:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:08:58Z" name: success-200-isvc-1bf2c-predictor.18be400c43558b28 namespace: kserve-ci-e2e-test resourceVersion: "25525" uid: eaf2d81c-9580-4ea8-9728-bcb069dddef5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:08:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1bf2c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25163" uid: f53dbaa4-9e5f-4e6e-bf9f-61ff0d690e3f kind: Event lastTimestamp: "2026-07-01T19:08:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:08:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:08:58Z" name: success-200-isvc-1bf2c-predictor.18be400c43c9dcc6 namespace: kserve-ci-e2e-test resourceVersion: "25528" uid: 8a0d6eaf-91b4-4f5f-b1ba-2f0689833134 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:07:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1bf2c namespace: kserve-ci-e2e-test resourceVersion: "24925" uid: c1e641e5-e9dc-4fb0-8152-f2d407789100 kind: Event lastTimestamp: "2026-07-01T19:08:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:07:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:08:46Z" name: success-200-isvc-1bf2c.18be4001c669d0cc namespace: kserve-ci-e2e-test resourceVersion: "25432" uid: 0dad5d34-a977-4c62-8c26-d1fc03c94564 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:07:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1bf2c namespace: kserve-ci-e2e-test resourceVersion: "24925" uid: c1e641e5-e9dc-4fb0-8152-f2d407789100 kind: Event lastTimestamp: "2026-07-01T19:07:58Z" message: 'Failed to update status for InferenceService "success-200-isvc-1bf2c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1bf2c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:07:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:07:58Z" name: success-200-isvc-1bf2c.18be4001c6fd1bd1 namespace: kserve-ci-e2e-test resourceVersion: "24939" uid: 2fba1caa-223c-475e-a00d-ef994446ee51 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:07:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1bf2c namespace: kserve-ci-e2e-test resourceVersion: "24925" uid: c1e641e5-e9dc-4fb0-8152-f2d407789100 kind: Event lastTimestamp: "2026-07-01T19:07:58Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1bf2c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:07:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:07:58Z" name: success-200-isvc-1bf2c.18be4001c6fd65c9 namespace: kserve-ci-e2e-test resourceVersion: "24940" uid: c19669ea-ab2e-4da3-931f-5f847cd8df69 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:08:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1bf2c namespace: kserve-ci-e2e-test resourceVersion: "25426" uid: c1e641e5-e9dc-4fb0-8152-f2d407789100 kind: Event lastTimestamp: "2026-07-01T19:08:46Z" message: InferenceService [success-200-isvc-1bf2c] is Ready metadata: creationTimestamp: "2026-07-01T19:08:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:08:46Z" name: success-200-isvc-1bf2c.18be400ceb78c131 namespace: kserve-ci-e2e-test resourceVersion: "25427" uid: 9a7086ae-c4af-4398-9862-0452c86b9ceb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:09:08.636174Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-43223-predictor-85967567b9-fkcgw namespace: kserve-ci-e2e-test resourceVersion: "25967" uid: a416bd65-9851-446c-a152-59fa2adab5d3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-43223-predictor-85967567b9-fkcgw to ip-10-0-129-245.ec2.internal metadata: creationTimestamp: "2026-07-01T19:09:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:09:08Z" name: success-200-isvc-43223-predictor-85967567b9-fkcgw.18be401205cca994 namespace: kserve-ci-e2e-test resourceVersion: "25972" uid: 9d93def3-96ea-466b-a7f7-88fd0a75bfb4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:08Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-43223-predictor-85967567b9-fkcgw namespace: kserve-ci-e2e-test resourceVersion: "25969" uid: a416bd65-9851-446c-a152-59fa2adab5d3 kind: Event lastTimestamp: "2026-07-01T19:09:08Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-43223-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:09:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:08Z" name: success-200-isvc-43223-predictor-85967567b9-fkcgw.18be401216bcca49 namespace: kserve-ci-e2e-test resourceVersion: "26025" uid: a73801d2-95aa-460c-83da-a1ba45fac8cb reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-43223-predictor-85967567b9-fkcgw namespace: kserve-ci-e2e-test resourceVersion: "25977" uid: a416bd65-9851-446c-a152-59fa2adab5d3 kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:09:09Z" name: success-200-isvc-43223-predictor-85967567b9-fkcgw.18be40124394c30d namespace: kserve-ci-e2e-test resourceVersion: "26154" uid: ec338d0c-8714-4a50-bfe8-82808b0911c5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-43223-predictor-85967567b9-fkcgw namespace: kserve-ci-e2e-test resourceVersion: "25969" uid: a416bd65-9851-446c-a152-59fa2adab5d3 kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:09Z" name: success-200-isvc-43223-predictor-85967567b9-fkcgw.18be4012453dbfeb namespace: kserve-ci-e2e-test resourceVersion: "26156" uid: de9d2f1f-c3cb-4e30-b807-33829172b057 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-43223-predictor-85967567b9-fkcgw namespace: kserve-ci-e2e-test resourceVersion: "25969" uid: a416bd65-9851-446c-a152-59fa2adab5d3 kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:09Z" name: success-200-isvc-43223-predictor-85967567b9-fkcgw.18be401248db60b4 namespace: kserve-ci-e2e-test resourceVersion: "26158" uid: 8dad0a8a-c5b0-43cb-9f94-d5d98a98a74d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-43223-predictor-85967567b9-fkcgw namespace: kserve-ci-e2e-test resourceVersion: "25969" uid: a416bd65-9851-446c-a152-59fa2adab5d3 kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:09Z" name: success-200-isvc-43223-predictor-85967567b9-fkcgw.18be401249523122 namespace: kserve-ci-e2e-test resourceVersion: "26159" uid: 64d04923-91ba-48b9-81c3-0320d6ff910a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-43223-predictor-85967567b9-fkcgw namespace: kserve-ci-e2e-test resourceVersion: "25969" uid: a416bd65-9851-446c-a152-59fa2adab5d3 kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:09Z" name: success-200-isvc-43223-predictor-85967567b9-fkcgw.18be4012495e61c7 namespace: kserve-ci-e2e-test resourceVersion: "26160" uid: 628aee69-31fc-4895-b25e-8bb3c2598a66 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-43223-predictor-85967567b9-fkcgw namespace: kserve-ci-e2e-test resourceVersion: "25969" uid: a416bd65-9851-446c-a152-59fa2adab5d3 kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:09Z" name: success-200-isvc-43223-predictor-85967567b9-fkcgw.18be40124da842d6 namespace: kserve-ci-e2e-test resourceVersion: "26164" uid: 42d060a6-ef35-4f39-b173-b92d26dec997 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-43223-predictor-85967567b9-fkcgw namespace: kserve-ci-e2e-test resourceVersion: "25969" uid: a416bd65-9851-446c-a152-59fa2adab5d3 kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:09Z" name: success-200-isvc-43223-predictor-85967567b9-fkcgw.18be40124e1d87fd namespace: kserve-ci-e2e-test resourceVersion: "26165" uid: f40a6343-9b7e-42dc-87aa-00b239e20d2f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-43223-predictor-85967567b9-fkcgw namespace: kserve-ci-e2e-test resourceVersion: "25969" uid: a416bd65-9851-446c-a152-59fa2adab5d3 kind: Event lastTimestamp: "2026-07-01T19:09:55Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:55Z" name: success-200-isvc-43223-predictor-85967567b9-fkcgw.18be4012501af91f namespace: kserve-ci-e2e-test resourceVersion: "26612" uid: 1bb8d017-5b76-46fc-a623-38840f345e48 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:26:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-43223-predictor-85967567b9-fkcgw namespace: kserve-ci-e2e-test resourceVersion: "25969" uid: a416bd65-9851-446c-a152-59fa2adab5d3 kind: Event lastTimestamp: "2026-07-01T19:26:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:26:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:26:38Z" name: success-200-isvc-43223-predictor-85967567b9-fkcgw.18be410665cac19b namespace: kserve-ci-e2e-test resourceVersion: "33695" uid: 52df242d-b0b8-42e5-a7bc-c7d3b4d760ee reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:26:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-43223-predictor-85967567b9-fkcgw namespace: kserve-ci-e2e-test resourceVersion: "25969" uid: a416bd65-9851-446c-a152-59fa2adab5d3 kind: Event lastTimestamp: "2026-07-01T19:26:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:26:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:26:38Z" name: success-200-isvc-43223-predictor-85967567b9-fkcgw.18be410665cb83d5 namespace: kserve-ci-e2e-test resourceVersion: "33696" uid: 6bd9d26e-6d5c-410f-a6cd-6014ad98921e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:26:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-43223-predictor-85967567b9-fkcgw namespace: kserve-ci-e2e-test resourceVersion: "25969" uid: a416bd65-9851-446c-a152-59fa2adab5d3 kind: Event lastTimestamp: "2026-07-01T19:26:40Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:26:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:26:40Z" name: success-200-isvc-43223-predictor-85967567b9-fkcgw.18be410705756a4e namespace: kserve-ci-e2e-test resourceVersion: "33764" uid: 0a2666a0-052b-4372-9fe5-14750532f7e8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-43223-predictor-85967567b9 namespace: kserve-ci-e2e-test resourceVersion: "25956" uid: 68fbde86-09e0-4dee-b9cc-e0e7ca09590a kind: Event lastTimestamp: "2026-07-01T19:09:08Z" message: 'Created pod: success-200-isvc-43223-predictor-85967567b9-fkcgw' metadata: creationTimestamp: "2026-07-01T19:09:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:09:08Z" name: success-200-isvc-43223-predictor-85967567b9.18be401205581919 namespace: kserve-ci-e2e-test resourceVersion: "25968" uid: 5876a6b0-cd2d-46fa-8b40-49b1ddcbf8a3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-43223-predictor namespace: kserve-ci-e2e-test resourceVersion: "25952" uid: b6aa2790-9a0a-4060-94fa-92d196a2459b kind: Event lastTimestamp: "2026-07-01T19:09:08Z" message: Scaled up replica set success-200-isvc-43223-predictor-85967567b9 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:09:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:09:08Z" name: success-200-isvc-43223-predictor.18be401203a910a2 namespace: kserve-ci-e2e-test resourceVersion: "25958" uid: c2ab3667-666a-4d9d-b8d8-a1621cf716cf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:09:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-43223-predictor namespace: kserve-ci-e2e-test resourceVersion: "25954" uid: bdb85bef-ef05-4197-8f63-8eda9f2bb899 kind: Event lastTimestamp: "2026-07-01T19:10:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:09:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:10:23Z" name: success-200-isvc-43223-predictor.18be401582407a10 namespace: kserve-ci-e2e-test resourceVersion: "26925" uid: d27c7e7a-f29b-4b25-a836-88c392f2bec6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:09:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-43223-predictor namespace: kserve-ci-e2e-test resourceVersion: "25954" uid: bdb85bef-ef05-4197-8f63-8eda9f2bb899 kind: Event lastTimestamp: "2026-07-01T19:10:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:09:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:10:23Z" name: success-200-isvc-43223-predictor.18be401582f5811e namespace: kserve-ci-e2e-test resourceVersion: "26926" uid: 2f930613-1672-40b8-82d9-cae59ca26aeb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-43223-predictor namespace: kserve-ci-e2e-test resourceVersion: "26329" uid: bdb85bef-ef05-4197-8f63-8eda9f2bb899 kind: Event lastTimestamp: "2026-07-01T19:09:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:09:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:09:38Z" name: success-200-isvc-43223-predictor.18be401902570a98 namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: affd21ce-92c8-4eb6-8762-3df6102f2bcf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-43223-predictor namespace: kserve-ci-e2e-test resourceVersion: "26329" uid: bdb85bef-ef05-4197-8f63-8eda9f2bb899 kind: Event lastTimestamp: "2026-07-01T19:09:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:09:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:09:38Z" name: success-200-isvc-43223-predictor.18be401902cdaeef namespace: kserve-ci-e2e-test resourceVersion: "26460" uid: 64c18186-c159-4489-981e-27a13430fa0b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:09:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-43223 namespace: kserve-ci-e2e-test resourceVersion: "25934" uid: eeb2516e-09fd-4c72-8caa-07d77cfbf42f kind: Event lastTimestamp: "2026-07-01T19:10:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:09:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:10:06Z" name: success-200-isvc-43223.18be401203d667c8 namespace: kserve-ci-e2e-test resourceVersion: "26715" uid: 1a2d5182-ebd6-41d1-b83d-98075f2c8ff6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-43223 namespace: kserve-ci-e2e-test resourceVersion: "25934" uid: eeb2516e-09fd-4c72-8caa-07d77cfbf42f kind: Event lastTimestamp: "2026-07-01T19:09:08Z" message: 'Failed to update status for InferenceService "success-200-isvc-43223": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-43223": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:09:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:09:08Z" name: success-200-isvc-43223.18be4012044179a5 namespace: kserve-ci-e2e-test resourceVersion: "25963" uid: c3968f6b-ecbe-4237-a0a0-be0e62a3e040 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-43223 namespace: kserve-ci-e2e-test resourceVersion: "25934" uid: eeb2516e-09fd-4c72-8caa-07d77cfbf42f kind: Event lastTimestamp: "2026-07-01T19:09:08Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-43223": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T19:09:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:09:08Z" name: success-200-isvc-43223.18be40120441d96b namespace: kserve-ci-e2e-test resourceVersion: "25966" uid: 5e5fc0b1-b4d7-498f-9ca3-08967ef5df6c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:10:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-43223 namespace: kserve-ci-e2e-test resourceVersion: "26712" uid: eeb2516e-09fd-4c72-8caa-07d77cfbf42f kind: Event lastTimestamp: "2026-07-01T19:10:05Z" message: InferenceService [success-200-isvc-43223] is Ready metadata: creationTimestamp: "2026-07-01T19:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:10:05Z" name: success-200-isvc-43223.18be401f6061a41f namespace: kserve-ci-e2e-test resourceVersion: "26714" uid: a9f6f0f3-2632-4430-981b-ba07cd734a0a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:09:09.163436Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4f643-predictor-785bf555c9-xsk4z namespace: kserve-ci-e2e-test resourceVersion: "26090" uid: 18547f9d-5826-4562-b935-5383beb63767 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4f643-predictor-785bf555c9-xsk4z to ip-10-0-129-245.ec2.internal metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:09:09Z" name: success-200-isvc-4f643-predictor-785bf555c9-xsk4z.18be4012253a00cf namespace: kserve-ci-e2e-test resourceVersion: "26098" uid: ef721185-866c-48f8-95c1-8a907f9601e1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4f643-predictor-785bf555c9-xsk4z namespace: kserve-ci-e2e-test resourceVersion: "26094" uid: 18547f9d-5826-4562-b935-5383beb63767 kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-4f643-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:09Z" name: success-200-isvc-4f643-predictor-785bf555c9-xsk4z.18be40122edbcc83 namespace: kserve-ci-e2e-test resourceVersion: "26135" uid: d6ef27fc-e72c-4a55-b69f-3512d2045bcc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:10Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4f643-predictor-785bf555c9-xsk4z namespace: kserve-ci-e2e-test resourceVersion: "26105" uid: 18547f9d-5826-4562-b935-5383beb63767 kind: Event lastTimestamp: "2026-07-01T19:09:10Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:09:10Z" name: success-200-isvc-4f643-predictor-785bf555c9-xsk4z.18be4012642ccbc7 namespace: kserve-ci-e2e-test resourceVersion: "26176" uid: b54a0ef4-d0c7-4efb-8458-399a51273b0a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4f643-predictor-785bf555c9-xsk4z namespace: kserve-ci-e2e-test resourceVersion: "26094" uid: 18547f9d-5826-4562-b935-5383beb63767 kind: Event lastTimestamp: "2026-07-01T19:09:10Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:10Z" name: success-200-isvc-4f643-predictor-785bf555c9-xsk4z.18be401265dc42ec namespace: kserve-ci-e2e-test resourceVersion: "26178" uid: 1162cf7c-fbe0-4f73-9a3f-1d2de7e6e5cf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4f643-predictor-785bf555c9-xsk4z namespace: kserve-ci-e2e-test resourceVersion: "26094" uid: 18547f9d-5826-4562-b935-5383beb63767 kind: Event lastTimestamp: "2026-07-01T19:09:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:10Z" name: success-200-isvc-4f643-predictor-785bf555c9-xsk4z.18be4012694d74bd namespace: kserve-ci-e2e-test resourceVersion: "26179" uid: c7dbe1c3-fdb9-4d61-b17c-88e66642cb5a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4f643-predictor-785bf555c9-xsk4z namespace: kserve-ci-e2e-test resourceVersion: "26094" uid: 18547f9d-5826-4562-b935-5383beb63767 kind: Event lastTimestamp: "2026-07-01T19:09:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:10Z" name: success-200-isvc-4f643-predictor-785bf555c9-xsk4z.18be401269bfbb3f namespace: kserve-ci-e2e-test resourceVersion: "26180" uid: 2683f968-80f1-4f62-a653-f254cd2d9bd6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4f643-predictor-785bf555c9-xsk4z namespace: kserve-ci-e2e-test resourceVersion: "26094" uid: 18547f9d-5826-4562-b935-5383beb63767 kind: Event lastTimestamp: "2026-07-01T19:09:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:10Z" name: success-200-isvc-4f643-predictor-785bf555c9-xsk4z.18be401269ca2140 namespace: kserve-ci-e2e-test resourceVersion: "26181" uid: 5f1dcb6f-ecd1-42e2-8aa1-fb1c2ce945f9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4f643-predictor-785bf555c9-xsk4z namespace: kserve-ci-e2e-test resourceVersion: "26094" uid: 18547f9d-5826-4562-b935-5383beb63767 kind: Event lastTimestamp: "2026-07-01T19:09:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:10Z" name: success-200-isvc-4f643-predictor-785bf555c9-xsk4z.18be4012718dab01 namespace: kserve-ci-e2e-test resourceVersion: "26187" uid: a0d11de7-519b-4770-b72d-4b87b1e198e7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4f643-predictor-785bf555c9-xsk4z namespace: kserve-ci-e2e-test resourceVersion: "26094" uid: 18547f9d-5826-4562-b935-5383beb63767 kind: Event lastTimestamp: "2026-07-01T19:09:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:10Z" name: success-200-isvc-4f643-predictor-785bf555c9-xsk4z.18be4012720715ce namespace: kserve-ci-e2e-test resourceVersion: "26189" uid: 660ee011-2c60-46e3-936a-be9929a257aa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T19:09:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4f643-predictor-785bf555c9-xsk4z namespace: kserve-ci-e2e-test resourceVersion: "26094" uid: 18547f9d-5826-4562-b935-5383beb63767 kind: Event lastTimestamp: "2026-07-01T19:18:26Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:09:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:18:26Z" name: success-200-isvc-4f643-predictor-785bf555c9-xsk4z.18be40128c716e9e namespace: kserve-ci-e2e-test resourceVersion: "30316" uid: 0630416b-e13a-48b5-aaf4-361d4c76032e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:18:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4f643-predictor-785bf555c9-xsk4z namespace: kserve-ci-e2e-test resourceVersion: "26094" uid: 18547f9d-5826-4562-b935-5383beb63767 kind: Event lastTimestamp: "2026-07-01T19:18:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:18:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:18:24Z" name: success-200-isvc-4f643-predictor-785bf555c9-xsk4z.18be409357c6f9b1 namespace: kserve-ci-e2e-test resourceVersion: "30274" uid: 416d6d73-8ec5-4dc8-8f82-f10a4f6bcc1d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:18:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4f643-predictor-785bf555c9-xsk4z namespace: kserve-ci-e2e-test resourceVersion: "26094" uid: 18547f9d-5826-4562-b935-5383beb63767 kind: Event lastTimestamp: "2026-07-01T19:18:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:18:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:18:24Z" name: success-200-isvc-4f643-predictor-785bf555c9-xsk4z.18be409357c769b8 namespace: kserve-ci-e2e-test resourceVersion: "30275" uid: 13a1f264-965e-48f9-bf6f-eb430646ca4b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:18:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4f643-predictor-785bf555c9-xsk4z namespace: kserve-ci-e2e-test resourceVersion: "26094" uid: 18547f9d-5826-4562-b935-5383beb63767 kind: Event lastTimestamp: "2026-07-01T19:18:26Z" message: 'Readiness probe failed: Get "https://10.134.0.39:8643/healthz": dial tcp 10.134.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:18:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:18:26Z" name: success-200-isvc-4f643-predictor-785bf555c9-xsk4z.18be40940111f226 namespace: kserve-ci-e2e-test resourceVersion: "30313" uid: 4ad23785-ffe5-41b7-98d9-61c9d7ec9edb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4f643-predictor-785bf555c9 namespace: kserve-ci-e2e-test resourceVersion: "26071" uid: 3d35e478-a6db-498e-a8c4-2b6eddc2ce37 kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: 'Created pod: success-200-isvc-4f643-predictor-785bf555c9-xsk4z' metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:09:09Z" name: success-200-isvc-4f643-predictor-785bf555c9.18be401224900977 namespace: kserve-ci-e2e-test resourceVersion: "26092" uid: c1e4531a-699f-4b0e-a416-12b77e30e72f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4f643-predictor namespace: kserve-ci-e2e-test resourceVersion: "26067" uid: 97bc5479-bcf0-450e-9fc9-910c01bd29a5 kind: Event lastTimestamp: "2026-07-01T19:09:09Z" message: Scaled up replica set success-200-isvc-4f643-predictor-785bf555c9 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:09:09Z" name: success-200-isvc-4f643-predictor.18be40122276538c namespace: kserve-ci-e2e-test resourceVersion: "26074" uid: c31e0b57-6ae6-4655-bfa1-b65ea8eb2882 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:09:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4f643-predictor namespace: kserve-ci-e2e-test resourceVersion: "26070" uid: 332d4406-89bc-4dd5-8d56-9a44a6ac86a8 kind: Event lastTimestamp: "2026-07-01T19:09:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:09:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:09:39Z" name: success-200-isvc-4f643-predictor.18be4015a12be80d namespace: kserve-ci-e2e-test resourceVersion: "26467" uid: d4b73005-957e-4d1b-9c4d-755e74c0405f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:09:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4f643-predictor namespace: kserve-ci-e2e-test resourceVersion: "26070" uid: 332d4406-89bc-4dd5-8d56-9a44a6ac86a8 kind: Event lastTimestamp: "2026-07-01T19:09:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:09:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:09:39Z" name: success-200-isvc-4f643-predictor.18be4015a19f907a namespace: kserve-ci-e2e-test resourceVersion: "26468" uid: 0f836d89-de3b-4014-9150-f277b5b7ddf2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:09:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4f643-predictor namespace: kserve-ci-e2e-test resourceVersion: "26339" uid: 332d4406-89bc-4dd5-8d56-9a44a6ac86a8 kind: Event lastTimestamp: "2026-07-01T19:10:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:09:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:10:24Z" name: success-200-isvc-4f643-predictor.18be401c9f32998d namespace: kserve-ci-e2e-test resourceVersion: "26933" uid: 43f1ac94-8f07-4c96-a13a-c1f1a04d4c59 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T19:09:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4f643-predictor namespace: kserve-ci-e2e-test resourceVersion: "26339" uid: 332d4406-89bc-4dd5-8d56-9a44a6ac86a8 kind: Event lastTimestamp: "2026-07-01T19:10:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:09:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:10:24Z" name: success-200-isvc-4f643-predictor.18be401c9fa85223 namespace: kserve-ci-e2e-test resourceVersion: "26934" uid: ad300141-fd02-4ea7-934e-faeb5ef33f5b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T19:09:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4f643 namespace: kserve-ci-e2e-test resourceVersion: "26047" uid: c7438e45-7069-4fde-b93a-171b1ac95cf6 kind: Event lastTimestamp: "2026-07-01T19:09:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T19:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:09:57Z" name: success-200-isvc-4f643.18be401222c113a9 namespace: kserve-ci-e2e-test resourceVersion: "26648" uid: 83997fd7-d669-4647-b754-c084caf6eb85 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4f643 namespace: kserve-ci-e2e-test resourceVersion: "26644" uid: c7438e45-7069-4fde-b93a-171b1ac95cf6 kind: Event lastTimestamp: "2026-07-01T19:09:57Z" message: InferenceService [success-200-isvc-4f643] is Ready metadata: creationTimestamp: "2026-07-01T19:09:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:09:57Z" name: success-200-isvc-4f643.18be401d4ef3f892 namespace: kserve-ci-e2e-test resourceVersion: "26645" uid: 8e0821b3-ccd4-42d7-b627-866a7615edd3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:58:29.484158Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-550f3-predictor-6564b49d87-mgttl namespace: kserve-ci-e2e-test resourceVersion: "20252" uid: baaaead7-e419-4611-a8f1-4b336faa25b0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-550f3-predictor-6564b49d87-mgttl to ip-10-0-129-245.ec2.internal metadata: creationTimestamp: "2026-07-01T18:58:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:58:29Z" name: success-200-isvc-550f3-predictor-6564b49d87-mgttl.18be3f7d355eeabf namespace: kserve-ci-e2e-test resourceVersion: "20257" uid: 6f26c4a4-e1b3-452f-8883-9a0bad6d33b8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:29Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-550f3-predictor-6564b49d87-mgttl namespace: kserve-ci-e2e-test resourceVersion: "20262" uid: baaaead7-e419-4611-a8f1-4b336faa25b0 kind: Event lastTimestamp: "2026-07-01T18:58:29Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:58:29Z" name: success-200-isvc-550f3-predictor-6564b49d87-mgttl.18be3f7d4ee0d0c1 namespace: kserve-ci-e2e-test resourceVersion: "20298" uid: b6efbf38-e6dc-4707-8e8f-3fac2114674a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-550f3-predictor-6564b49d87-mgttl namespace: kserve-ci-e2e-test resourceVersion: "20254" uid: baaaead7-e419-4611-a8f1-4b336faa25b0 kind: Event lastTimestamp: "2026-07-01T18:58:29Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T18:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:29Z" name: success-200-isvc-550f3-predictor-6564b49d87-mgttl.18be3f7d50941a0f namespace: kserve-ci-e2e-test resourceVersion: "20300" uid: 2fa5c3ba-914b-4718-89a0-399f973ce585 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-550f3-predictor-6564b49d87-mgttl namespace: kserve-ci-e2e-test resourceVersion: "20254" uid: baaaead7-e419-4611-a8f1-4b336faa25b0 kind: Event lastTimestamp: "2026-07-01T18:58:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:30Z" name: success-200-isvc-550f3-predictor-6564b49d87-mgttl.18be3f7d5432e92f namespace: kserve-ci-e2e-test resourceVersion: "20301" uid: 7b25f51e-89e6-4de8-80f1-927bf9a8ae09 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-550f3-predictor-6564b49d87-mgttl namespace: kserve-ci-e2e-test resourceVersion: "20254" uid: baaaead7-e419-4611-a8f1-4b336faa25b0 kind: Event lastTimestamp: "2026-07-01T18:58:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:30Z" name: success-200-isvc-550f3-predictor-6564b49d87-mgttl.18be3f7d54ca5035 namespace: kserve-ci-e2e-test resourceVersion: "20302" uid: b109cade-c57d-4495-b66f-a539f38d7856 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-550f3-predictor-6564b49d87-mgttl namespace: kserve-ci-e2e-test resourceVersion: "20254" uid: baaaead7-e419-4611-a8f1-4b336faa25b0 kind: Event lastTimestamp: "2026-07-01T18:58:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:30Z" name: success-200-isvc-550f3-predictor-6564b49d87-mgttl.18be3f7d54d688f9 namespace: kserve-ci-e2e-test resourceVersion: "20305" uid: 433ffa25-e4ab-4544-8f73-5d8d52ec0ca9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-550f3-predictor-6564b49d87-mgttl namespace: kserve-ci-e2e-test resourceVersion: "20254" uid: baaaead7-e419-4611-a8f1-4b336faa25b0 kind: Event lastTimestamp: "2026-07-01T18:58:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:30Z" name: success-200-isvc-550f3-predictor-6564b49d87-mgttl.18be3f7d578ca6d1 namespace: kserve-ci-e2e-test resourceVersion: "20309" uid: 6e3019f9-1c08-4293-8ec2-de4d151cbd71 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-550f3-predictor-6564b49d87-mgttl namespace: kserve-ci-e2e-test resourceVersion: "20254" uid: baaaead7-e419-4611-a8f1-4b336faa25b0 kind: Event lastTimestamp: "2026-07-01T18:58:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:30Z" name: success-200-isvc-550f3-predictor-6564b49d87-mgttl.18be3f7d580150fd namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: 128951fc-e579-428e-87cc-4039c5d8654c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-01T18:58:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-550f3-predictor-6564b49d87-mgttl namespace: kserve-ci-e2e-test resourceVersion: "20254" uid: baaaead7-e419-4611-a8f1-4b336faa25b0 kind: Event lastTimestamp: "2026-07-01T18:59:56Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:56Z" name: success-200-isvc-550f3-predictor-6564b49d87-mgttl.18be3f7d69dcc76a namespace: kserve-ci-e2e-test resourceVersion: "21487" uid: 3aa2c8cf-4d15-4c12-a41b-21161291a05c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-550f3-predictor-6564b49d87-mgttl namespace: kserve-ci-e2e-test resourceVersion: "20254" uid: baaaead7-e419-4611-a8f1-4b336faa25b0 kind: Event lastTimestamp: "2026-07-01T18:59:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:53Z" name: success-200-isvc-550f3-predictor-6564b49d87-mgttl.18be3f90dabb8681 namespace: kserve-ci-e2e-test resourceVersion: "21407" uid: 007b2497-cd6f-4aa1-9973-0d3f502903b9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-550f3-predictor-6564b49d87-mgttl namespace: kserve-ci-e2e-test resourceVersion: "20254" uid: baaaead7-e419-4611-a8f1-4b336faa25b0 kind: Event lastTimestamp: "2026-07-01T18:59:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:53Z" name: success-200-isvc-550f3-predictor-6564b49d87-mgttl.18be3f90dabbf584 namespace: kserve-ci-e2e-test resourceVersion: "21411" uid: 85da1654-652d-49f2-85c4-f6ff4c758249 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-550f3-predictor-6564b49d87-mgttl namespace: kserve-ci-e2e-test resourceVersion: "20254" uid: baaaead7-e419-4611-a8f1-4b336faa25b0 kind: Event lastTimestamp: "2026-07-01T18:59:56Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:56Z" name: success-200-isvc-550f3-predictor-6564b49d87-mgttl.18be3f91701ce0c2 namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 9b997cd4-7f98-49d0-9b53-38c40c9ce211 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-550f3-predictor-6564b49d87 namespace: kserve-ci-e2e-test resourceVersion: "20238" uid: 5d169146-7ef2-4eb3-9600-e8c6c296a765 kind: Event lastTimestamp: "2026-07-01T18:58:29Z" message: 'Created pod: success-200-isvc-550f3-predictor-6564b49d87-mgttl' metadata: creationTimestamp: "2026-07-01T18:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:58:29Z" name: success-200-isvc-550f3-predictor-6564b49d87.18be3f7d34e44e21 namespace: kserve-ci-e2e-test resourceVersion: "20253" uid: 84928786-4ce3-4859-a683-edfff3da8970 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-550f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20231" uid: e0eaa11f-d06a-4ba7-ab89-40bed4b2aec6 kind: Event lastTimestamp: "2026-07-01T18:58:29Z" message: Scaled up replica set success-200-isvc-550f3-predictor-6564b49d87 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:58:29Z" name: success-200-isvc-550f3-predictor.18be3f7d329d5fe5 namespace: kserve-ci-e2e-test resourceVersion: "20241" uid: 88238728-aa86-442f-9e14-d7ec9d8efea9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:58:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-550f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20236" uid: bc242c10-0964-4b82-8bad-d1990a4f250c kind: Event lastTimestamp: "2026-07-01T18:58:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:58:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:58:59Z" name: success-200-isvc-550f3-predictor.18be3f80b14a202c namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: 0b402a10-9e63-4679-8f25-23a37da4e941 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:58:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-550f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20236" uid: bc242c10-0964-4b82-8bad-d1990a4f250c kind: Event lastTimestamp: "2026-07-01T18:58:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:58:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:58:59Z" name: success-200-isvc-550f3-predictor.18be3f80b1bb7b1b namespace: kserve-ci-e2e-test resourceVersion: "20768" uid: 44b1ef01-7a2d-43a2-b44f-756ee1f1e6b4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:59:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-550f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20604" uid: bc242c10-0964-4b82-8bad-d1990a4f250c kind: Event lastTimestamp: "2026-07-01T18:59:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:59:44Z" name: success-200-isvc-550f3-predictor.18be3f87af6392ea namespace: kserve-ci-e2e-test resourceVersion: "21218" uid: f1fe88a0-b21a-40eb-b1e3-c46f2c8e17c3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:59:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-550f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20604" uid: bc242c10-0964-4b82-8bad-d1990a4f250c kind: Event lastTimestamp: "2026-07-01T18:59:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:59:44Z" name: success-200-isvc-550f3-predictor.18be3f87afdb9f44 namespace: kserve-ci-e2e-test resourceVersion: "21219" uid: 18f484eb-f07f-4417-998a-8cfe39e997d2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:58:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-550f3 namespace: kserve-ci-e2e-test resourceVersion: "20204" uid: 62dd3526-f393-460e-9c92-5ad45fa50c17 kind: Event lastTimestamp: "2026-07-01T18:59:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:59:26Z" name: success-200-isvc-550f3.18be3f7d33108788 namespace: kserve-ci-e2e-test resourceVersion: "20998" uid: 633b5796-a334-4094-88d2-95d20c1d08f6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-550f3 namespace: kserve-ci-e2e-test resourceVersion: "20204" uid: 62dd3526-f393-460e-9c92-5ad45fa50c17 kind: Event lastTimestamp: "2026-07-01T18:58:29Z" message: 'Failed to update status for InferenceService "success-200-isvc-550f3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-550f3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:58:29Z" name: success-200-isvc-550f3.18be3f7d33841940 namespace: kserve-ci-e2e-test resourceVersion: "20247" uid: eb5f7bd5-ff8a-4f8e-85f3-e7beef704578 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-550f3 namespace: kserve-ci-e2e-test resourceVersion: "20204" uid: 62dd3526-f393-460e-9c92-5ad45fa50c17 kind: Event lastTimestamp: "2026-07-01T18:58:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-550f3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:58:29Z" name: success-200-isvc-550f3.18be3f7d338479b4 namespace: kserve-ci-e2e-test resourceVersion: "20250" uid: 73cdb4da-3d6d-44ef-abf6-bca3674a98c4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-550f3 namespace: kserve-ci-e2e-test resourceVersion: "20995" uid: 62dd3526-f393-460e-9c92-5ad45fa50c17 kind: Event lastTimestamp: "2026-07-01T18:59:26Z" message: InferenceService [success-200-isvc-550f3] is Ready metadata: creationTimestamp: "2026-07-01T18:59:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:59:26Z" name: success-200-isvc-550f3.18be3f8a7a0c681f namespace: kserve-ci-e2e-test resourceVersion: "20997" uid: 86acc0f0-64cd-4110-a951-f5c37fd51f03 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:48:18.789728Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9857d-predictor-c6ddb6cb7-54ljt namespace: kserve-ci-e2e-test resourceVersion: "15224" uid: b4f0fd3f-76a8-46ca-857d-8b4e1f662afe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-9857d-predictor-c6ddb6cb7-54ljt to ip-10-0-129-245.ec2.internal metadata: creationTimestamp: "2026-07-01T18:48:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:48:18Z" name: success-200-isvc-9857d-predictor-c6ddb6cb7-54ljt.18be3eef05256e6d namespace: kserve-ci-e2e-test resourceVersion: "15229" uid: ad2ad3e3-67f5-4a4e-a2ee-e97ce70a3f12 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:18Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9857d-predictor-c6ddb6cb7-54ljt namespace: kserve-ci-e2e-test resourceVersion: "15227" uid: b4f0fd3f-76a8-46ca-857d-8b4e1f662afe kind: Event lastTimestamp: "2026-07-01T18:48:18Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-9857d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:18Z" name: success-200-isvc-9857d-predictor-c6ddb6cb7-54ljt.18be3eef10f7e0d6 namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: bcff80e2-0a8d-47b6-82f0-c886e7cdc41c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:19Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9857d-predictor-c6ddb6cb7-54ljt namespace: kserve-ci-e2e-test resourceVersion: "15246" uid: b4f0fd3f-76a8-46ca-857d-8b4e1f662afe kind: Event lastTimestamp: "2026-07-01T18:48:19Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:48:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:48:19Z" name: success-200-isvc-9857d-predictor-c6ddb6cb7-54ljt.18be3eef4296d69a namespace: kserve-ci-e2e-test resourceVersion: "15317" uid: 0aebf841-3889-40b8-b135-a2c8e840b5f8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9857d-predictor-c6ddb6cb7-54ljt namespace: kserve-ci-e2e-test resourceVersion: "15227" uid: b4f0fd3f-76a8-46ca-857d-8b4e1f662afe kind: Event lastTimestamp: "2026-07-01T18:48:19Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T18:48:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:19Z" name: success-200-isvc-9857d-predictor-c6ddb6cb7-54ljt.18be3eef444691d7 namespace: kserve-ci-e2e-test resourceVersion: "15319" uid: d5975ef2-6e26-4208-aa63-56be722ad4cb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9857d-predictor-c6ddb6cb7-54ljt namespace: kserve-ci-e2e-test resourceVersion: "15227" uid: b4f0fd3f-76a8-46ca-857d-8b4e1f662afe kind: Event lastTimestamp: "2026-07-01T18:48:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:48:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:19Z" name: success-200-isvc-9857d-predictor-c6ddb6cb7-54ljt.18be3eef477721c2 namespace: kserve-ci-e2e-test resourceVersion: "15320" uid: acfa90c0-3683-4e1a-9fe3-b01b21b3f734 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9857d-predictor-c6ddb6cb7-54ljt namespace: kserve-ci-e2e-test resourceVersion: "15227" uid: b4f0fd3f-76a8-46ca-857d-8b4e1f662afe kind: Event lastTimestamp: "2026-07-01T18:48:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:48:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:19Z" name: success-200-isvc-9857d-predictor-c6ddb6cb7-54ljt.18be3eef47e39461 namespace: kserve-ci-e2e-test resourceVersion: "15321" uid: c9518f9a-ea82-4d23-b52d-549292f1be3e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9857d-predictor-c6ddb6cb7-54ljt namespace: kserve-ci-e2e-test resourceVersion: "15227" uid: b4f0fd3f-76a8-46ca-857d-8b4e1f662afe kind: Event lastTimestamp: "2026-07-01T18:48:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:48:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:19Z" name: success-200-isvc-9857d-predictor-c6ddb6cb7-54ljt.18be3eef47ec6aed namespace: kserve-ci-e2e-test resourceVersion: "15322" uid: c4817ac7-df37-4399-ba13-c696969a69c0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9857d-predictor-c6ddb6cb7-54ljt namespace: kserve-ci-e2e-test resourceVersion: "15227" uid: b4f0fd3f-76a8-46ca-857d-8b4e1f662afe kind: Event lastTimestamp: "2026-07-01T18:48:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:48:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:19Z" name: success-200-isvc-9857d-predictor-c6ddb6cb7-54ljt.18be3eef4abb3f80 namespace: kserve-ci-e2e-test resourceVersion: "15324" uid: 71e97ec7-604a-4896-a9fc-f3820393f614 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9857d-predictor-c6ddb6cb7-54ljt namespace: kserve-ci-e2e-test resourceVersion: "15227" uid: b4f0fd3f-76a8-46ca-857d-8b4e1f662afe kind: Event lastTimestamp: "2026-07-01T18:48:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:48:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:19Z" name: success-200-isvc-9857d-predictor-c6ddb6cb7-54ljt.18be3eef4b2bd387 namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: abfded33-29e6-4d66-8a92-a9af0b3491fa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:48:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9857d-predictor-c6ddb6cb7-54ljt namespace: kserve-ci-e2e-test resourceVersion: "15227" uid: b4f0fd3f-76a8-46ca-857d-8b4e1f662afe kind: Event lastTimestamp: "2026-07-01T18:48:57Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:57Z" name: success-200-isvc-9857d-predictor-c6ddb6cb7-54ljt.18be3eef8c74b856 namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 1576047d-8e93-4a8e-a1bb-35f4969449db reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9857d-predictor-c6ddb6cb7-54ljt namespace: kserve-ci-e2e-test resourceVersion: "15227" uid: b4f0fd3f-76a8-46ca-857d-8b4e1f662afe kind: Event lastTimestamp: "2026-07-01T18:57:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:57:33Z" name: success-200-isvc-9857d-predictor-c6ddb6cb7-54ljt.18be3f7035bd4508 namespace: kserve-ci-e2e-test resourceVersion: "19662" uid: 211e31ab-fa40-4f18-bd31-5ec6b44b9c06 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9857d-predictor-c6ddb6cb7-54ljt namespace: kserve-ci-e2e-test resourceVersion: "15227" uid: b4f0fd3f-76a8-46ca-857d-8b4e1f662afe kind: Event lastTimestamp: "2026-07-01T18:57:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:57:33Z" name: success-200-isvc-9857d-predictor-c6ddb6cb7-54ljt.18be3f7035be49f9 namespace: kserve-ci-e2e-test resourceVersion: "19665" uid: bb97601a-532a-4433-9973-be5a9d83ba28 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-9857d-predictor-c6ddb6cb7 namespace: kserve-ci-e2e-test resourceVersion: "15216" uid: 65896512-9c7b-48fb-92b7-7002f0e6fc12 kind: Event lastTimestamp: "2026-07-01T18:48:18Z" message: 'Created pod: success-200-isvc-9857d-predictor-c6ddb6cb7-54ljt' metadata: creationTimestamp: "2026-07-01T18:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:48:18Z" name: success-200-isvc-9857d-predictor-c6ddb6cb7.18be3eef04b17b22 namespace: kserve-ci-e2e-test resourceVersion: "15226" uid: 48143d0a-669c-4c51-b6c9-65373f9db8ac reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-9857d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15214" uid: 5f54c6ff-9071-48b1-8698-59c003248e19 kind: Event lastTimestamp: "2026-07-01T18:48:18Z" message: Scaled up replica set success-200-isvc-9857d-predictor-c6ddb6cb7 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:48:18Z" name: success-200-isvc-9857d-predictor.18be3eef01a36d9e namespace: kserve-ci-e2e-test resourceVersion: "15217" uid: a61feb4e-8200-4df6-a1fc-31f0cfc3abf1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:48:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9857d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15215" uid: a6ec4e74-b65c-438e-9880-e6c3813cd4e1 kind: Event lastTimestamp: "2026-07-01T18:49:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:48:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:49:03Z" name: success-200-isvc-9857d-predictor.18be3ef28060732d namespace: kserve-ci-e2e-test resourceVersion: "15718" uid: e8eac321-b4a3-4262-b611-552d6d7de981 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:48:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9857d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15215" uid: a6ec4e74-b65c-438e-9880-e6c3813cd4e1 kind: Event lastTimestamp: "2026-07-01T18:49:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:48:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:49:03Z" name: success-200-isvc-9857d-predictor.18be3ef280cd8d11 namespace: kserve-ci-e2e-test resourceVersion: "15719" uid: c626b07d-9a87-4eca-b284-72d22b8100f5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:49:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9857d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15459" uid: a6ec4e74-b65c-438e-9880-e6c3813cd4e1 kind: Event lastTimestamp: "2026-07-01T18:49:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:49:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:49:48Z" name: success-200-isvc-9857d-predictor.18be3efcfd090127 namespace: kserve-ci-e2e-test resourceVersion: "16342" uid: 6f1fc0fb-5134-4db6-9e77-59ca1f8fe906 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:49:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9857d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15459" uid: a6ec4e74-b65c-438e-9880-e6c3813cd4e1 kind: Event lastTimestamp: "2026-07-01T18:49:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:49:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:49:48Z" name: success-200-isvc-9857d-predictor.18be3efcfda2c665 namespace: kserve-ci-e2e-test resourceVersion: "16343" uid: 3c1ce6db-9993-492d-b58e-c14aa5e3f277 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:48:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9857d namespace: kserve-ci-e2e-test resourceVersion: "15205" uid: 860e0224-5d07-455c-820f-6e22f0351651 kind: Event lastTimestamp: "2026-07-01T18:49:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:49:07Z" name: success-200-isvc-9857d.18be3eef01a714ed namespace: kserve-ci-e2e-test resourceVersion: "15803" uid: f39f407e-7a62-43be-93cc-e386b6a0c694 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9857d namespace: kserve-ci-e2e-test resourceVersion: "15205" uid: 860e0224-5d07-455c-820f-6e22f0351651 kind: Event lastTimestamp: "2026-07-01T18:48:18Z" message: 'Failed to update status for InferenceService "success-200-isvc-9857d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-9857d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:48:18Z" name: success-200-isvc-9857d.18be3eef0225191c namespace: kserve-ci-e2e-test resourceVersion: "15220" uid: 6bdbd535-2028-4263-975d-2804702995f4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9857d namespace: kserve-ci-e2e-test resourceVersion: "15205" uid: 860e0224-5d07-455c-820f-6e22f0351651 kind: Event lastTimestamp: "2026-07-01T18:48:18Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-9857d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:48:18Z" name: success-200-isvc-9857d.18be3eef02256009 namespace: kserve-ci-e2e-test resourceVersion: "15221" uid: f2ef5670-1bc5-49c0-88d8-ae4011ba306b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9857d namespace: kserve-ci-e2e-test resourceVersion: "15801" uid: 860e0224-5d07-455c-820f-6e22f0351651 kind: Event lastTimestamp: "2026-07-01T18:49:07Z" message: InferenceService [success-200-isvc-9857d] is Ready metadata: creationTimestamp: "2026-07-01T18:49:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:49:07Z" name: success-200-isvc-9857d.18be3efa48568e27 namespace: kserve-ci-e2e-test resourceVersion: "15802" uid: 599ba56f-53e3-415f-9b49-a9a7c0e8a4c0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:58:43.900234Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: 7f36625f-48ef-4d40-94d1-85d3370da2d4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-af621-predictor-688d7d5cb9-p4nrw to ip-10-0-129-245.ec2.internal metadata: creationTimestamp: "2026-07-01T18:58:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:58:43Z" name: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw.18be3f8090a2b1bd namespace: kserve-ci-e2e-test resourceVersion: "20553" uid: acf6960e-863f-4130-a3d8-91d791114e12 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:44Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: 7f36625f-48ef-4d40-94d1-85d3370da2d4 kind: Event lastTimestamp: "2026-07-01T18:58:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-af621-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:58:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:44Z" name: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw.18be3f809baf7af4 namespace: kserve-ci-e2e-test resourceVersion: "20592" uid: f2615c4d-374f-4f37-bb9b-7a29436ef55d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:44Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw namespace: kserve-ci-e2e-test resourceVersion: "20560" uid: 7f36625f-48ef-4d40-94d1-85d3370da2d4 kind: Event lastTimestamp: "2026-07-01T18:58:44Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:58:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:58:44Z" name: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw.18be3f80ce87080c namespace: kserve-ci-e2e-test resourceVersion: "20612" uid: 365a0f69-f0cc-47f1-8ecb-9c497b33ec4d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: 7f36625f-48ef-4d40-94d1-85d3370da2d4 kind: Event lastTimestamp: "2026-07-01T18:58:44Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T18:58:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:44Z" name: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw.18be3f80cff0c221 namespace: kserve-ci-e2e-test resourceVersion: "20615" uid: af85fb52-c524-4202-b0e8-bea8df0f9e80 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: 7f36625f-48ef-4d40-94d1-85d3370da2d4 kind: Event lastTimestamp: "2026-07-01T18:58:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:45Z" name: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw.18be3f80d34b4206 namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: 95e2708d-ac95-47d8-8bf4-2ddb35b56d79 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: 7f36625f-48ef-4d40-94d1-85d3370da2d4 kind: Event lastTimestamp: "2026-07-01T18:58:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:45Z" name: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw.18be3f80d3c6c8cb namespace: kserve-ci-e2e-test resourceVersion: "20618" uid: e7c44236-1030-46ca-a44e-d0f536557aca reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: 7f36625f-48ef-4d40-94d1-85d3370da2d4 kind: Event lastTimestamp: "2026-07-01T18:58:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:45Z" name: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw.18be3f80d3d2b8a8 namespace: kserve-ci-e2e-test resourceVersion: "20619" uid: f0252b27-c552-4a82-a455-9456d56c7d8b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: 7f36625f-48ef-4d40-94d1-85d3370da2d4 kind: Event lastTimestamp: "2026-07-01T18:58:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:45Z" name: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw.18be3f80d72eedcb namespace: kserve-ci-e2e-test resourceVersion: "20623" uid: 81b87564-4e6a-4e58-aebd-f633f3d1fba6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: 7f36625f-48ef-4d40-94d1-85d3370da2d4 kind: Event lastTimestamp: "2026-07-01T18:58:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:45Z" name: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw.18be3f80d7b09514 namespace: kserve-ci-e2e-test resourceVersion: "20624" uid: d5a26ab9-2243-4e8f-9585-36d94abf53a2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-01T18:58:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: 7f36625f-48ef-4d40-94d1-85d3370da2d4 kind: Event lastTimestamp: "2026-07-01T19:08:02Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:08:02Z" name: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw.18be3f80ec89071a namespace: kserve-ci-e2e-test resourceVersion: "25073" uid: 0620b4a8-ede7-4d7a-ac1d-f6c03a28c2ba reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:07:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: 7f36625f-48ef-4d40-94d1-85d3370da2d4 kind: Event lastTimestamp: "2026-07-01T19:07:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:07:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:07:58Z" name: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw.18be4001cb7d74cb namespace: kserve-ci-e2e-test resourceVersion: "24962" uid: 44b54413-777b-414c-9d4d-01618395dcad reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:07:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: 7f36625f-48ef-4d40-94d1-85d3370da2d4 kind: Event lastTimestamp: "2026-07-01T19:07:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:07:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:07:58Z" name: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw.18be4001cb80afce namespace: kserve-ci-e2e-test resourceVersion: "24965" uid: 7ac59e41-f0cc-4ff4-8a5b-dd69f83141a1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:08:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: 7f36625f-48ef-4d40-94d1-85d3370da2d4 kind: Event lastTimestamp: "2026-07-01T19:08:02Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T19:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:08:02Z" name: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw.18be40029d291b31 namespace: kserve-ci-e2e-test resourceVersion: "25072" uid: bac07f78-5d74-4128-a4eb-72aec6392504 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-af621-predictor-688d7d5cb9 namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: 7f303324-12bf-4f22-863b-fc26f441b48a kind: Event lastTimestamp: "2026-07-01T18:58:43Z" message: 'Created pod: success-200-isvc-af621-predictor-688d7d5cb9-p4nrw' metadata: creationTimestamp: "2026-07-01T18:58:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:58:43Z" name: success-200-isvc-af621-predictor-688d7d5cb9.18be3f80901f7640 namespace: kserve-ci-e2e-test resourceVersion: "20547" uid: 763f93a7-d473-41dc-9bc4-13aa2e75ce8a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-af621-predictor namespace: kserve-ci-e2e-test resourceVersion: "20524" uid: 0f2f4a81-d01f-452f-94b7-573c6a76d95f kind: Event lastTimestamp: "2026-07-01T18:58:43Z" message: Scaled up replica set success-200-isvc-af621-predictor-688d7d5cb9 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:58:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:58:43Z" name: success-200-isvc-af621-predictor.18be3f808e233da5 namespace: kserve-ci-e2e-test resourceVersion: "20535" uid: 09c40de7-0b22-412a-aab0-ab92798167c0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:58:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-af621-predictor namespace: kserve-ci-e2e-test resourceVersion: "20528" uid: d13f0290-43ac-480b-82e0-0f9f5c524790 kind: Event lastTimestamp: "2026-07-01T18:59:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:59:13Z" name: success-200-isvc-af621-predictor.18be3f840d832e8e namespace: kserve-ci-e2e-test resourceVersion: "20879" uid: c882edf8-656b-48b4-953e-b8bfab84ca33 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:58:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-af621-predictor namespace: kserve-ci-e2e-test resourceVersion: "20528" uid: d13f0290-43ac-480b-82e0-0f9f5c524790 kind: Event lastTimestamp: "2026-07-01T18:59:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:59:13Z" name: success-200-isvc-af621-predictor.18be3f840df74147 namespace: kserve-ci-e2e-test resourceVersion: "20880" uid: 09abcca9-ffdf-426f-8400-8448caabe5d6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:59:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-af621-predictor namespace: kserve-ci-e2e-test resourceVersion: "20755" uid: d13f0290-43ac-480b-82e0-0f9f5c524790 kind: Event lastTimestamp: "2026-07-01T18:59:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:59:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:59:58Z" name: success-200-isvc-af621-predictor.18be3f8b0c8bf114 namespace: kserve-ci-e2e-test resourceVersion: "21516" uid: e59b30ab-2095-46da-9b8b-2a3b66f9b160 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:59:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-af621-predictor namespace: kserve-ci-e2e-test resourceVersion: "20755" uid: d13f0290-43ac-480b-82e0-0f9f5c524790 kind: Event lastTimestamp: "2026-07-01T18:59:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:59:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:59:58Z" name: success-200-isvc-af621-predictor.18be3f8b0d022ed7 namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: 5d080434-18d6-4ce5-800d-96631b5bacb7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:58:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-af621 namespace: kserve-ci-e2e-test resourceVersion: "20505" uid: 155f9bdf-7cb2-43d5-8f08-4ccad823a7f4 kind: Event lastTimestamp: "2026-07-01T18:59:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:58:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:59:32Z" name: success-200-isvc-af621.18be3f808e255f34 namespace: kserve-ci-e2e-test resourceVersion: "21068" uid: ca726204-a938-499f-91ed-e30a34f7e863 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-af621 namespace: kserve-ci-e2e-test resourceVersion: "21062" uid: 155f9bdf-7cb2-43d5-8f08-4ccad823a7f4 kind: Event lastTimestamp: "2026-07-01T18:59:32Z" message: InferenceService [success-200-isvc-af621] is Ready metadata: creationTimestamp: "2026-07-01T18:59:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:59:32Z" name: success-200-isvc-af621.18be3f8be6899d98 namespace: kserve-ci-e2e-test resourceVersion: "21063" uid: 9d4318cb-9755-4a43-9ecd-a80d511f7c71 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:57:33.647147Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5 namespace: kserve-ci-e2e-test resourceVersion: "19649" uid: d26e74b3-f903-4028-bf9f-ec2bb2dea9a8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5 to ip-10-0-129-245.ec2.internal metadata: creationTimestamp: "2026-07-01T18:57:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:57:33Z" name: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5.18be3f703539aade namespace: kserve-ci-e2e-test resourceVersion: "19658" uid: e075c2a3-178d-49a4-baed-b7485f958536 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:33Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5 namespace: kserve-ci-e2e-test resourceVersion: "19653" uid: d26e74b3-f903-4028-bf9f-ec2bb2dea9a8 kind: Event lastTimestamp: "2026-07-01T18:57:33Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-c059b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:57:33Z" name: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5.18be3f704119e330 namespace: kserve-ci-e2e-test resourceVersion: "19709" uid: d3044774-b4ea-4b85-b1fc-c579ed2cb479 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:34Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5 namespace: kserve-ci-e2e-test resourceVersion: "19671" uid: d26e74b3-f903-4028-bf9f-ec2bb2dea9a8 kind: Event lastTimestamp: "2026-07-01T18:57:34Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:57:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:57:34Z" name: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5.18be3f7073764c8f namespace: kserve-ci-e2e-test resourceVersion: "19725" uid: 7fa88523-576b-4553-a172-6157f927b22a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5 namespace: kserve-ci-e2e-test resourceVersion: "19653" uid: d26e74b3-f903-4028-bf9f-ec2bb2dea9a8 kind: Event lastTimestamp: "2026-07-01T18:57:34Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T18:57:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:57:34Z" name: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5.18be3f70750b8ea1 namespace: kserve-ci-e2e-test resourceVersion: "19727" uid: e8de2479-965c-41ff-acbb-22cce9beac9c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5 namespace: kserve-ci-e2e-test resourceVersion: "19653" uid: d26e74b3-f903-4028-bf9f-ec2bb2dea9a8 kind: Event lastTimestamp: "2026-07-01T18:57:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:57:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:57:34Z" name: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5.18be3f7079860f09 namespace: kserve-ci-e2e-test resourceVersion: "19731" uid: 74565d35-3e95-4936-a36a-06d0b462e264 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5 namespace: kserve-ci-e2e-test resourceVersion: "19653" uid: d26e74b3-f903-4028-bf9f-ec2bb2dea9a8 kind: Event lastTimestamp: "2026-07-01T18:57:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:57:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:57:34Z" name: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5.18be3f707a018263 namespace: kserve-ci-e2e-test resourceVersion: "19733" uid: ecdc1c77-91dc-4851-8fd4-85b5f4135f51 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5 namespace: kserve-ci-e2e-test resourceVersion: "19653" uid: d26e74b3-f903-4028-bf9f-ec2bb2dea9a8 kind: Event lastTimestamp: "2026-07-01T18:57:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:57:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:57:34Z" name: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5.18be3f707a0b69ab namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: 228a42f8-5544-4b69-aaa1-df3e23a654b3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5 namespace: kserve-ci-e2e-test resourceVersion: "19653" uid: d26e74b3-f903-4028-bf9f-ec2bb2dea9a8 kind: Event lastTimestamp: "2026-07-01T18:57:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:57:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:57:34Z" name: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5.18be3f707d091f03 namespace: kserve-ci-e2e-test resourceVersion: "19735" uid: d35d2bfe-ed49-4e99-846c-62c94a4de23c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5 namespace: kserve-ci-e2e-test resourceVersion: "19653" uid: d26e74b3-f903-4028-bf9f-ec2bb2dea9a8 kind: Event lastTimestamp: "2026-07-01T18:57:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:57:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:57:34Z" name: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5.18be3f707d7ab743 namespace: kserve-ci-e2e-test resourceVersion: "19737" uid: 1d16abc2-c5f8-4c52-a4e7-3bb7de4ffd8e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:57:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5 namespace: kserve-ci-e2e-test resourceVersion: "19653" uid: d26e74b3-f903-4028-bf9f-ec2bb2dea9a8 kind: Event lastTimestamp: "2026-07-01T18:58:11Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:57:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:11Z" name: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5.18be3f708db937d7 namespace: kserve-ci-e2e-test resourceVersion: "20035" uid: 76297374-3a88-4082-bd3c-d1984bf820a1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5 namespace: kserve-ci-e2e-test resourceVersion: "19653" uid: d26e74b3-f903-4028-bf9f-ec2bb2dea9a8 kind: Event lastTimestamp: "2026-07-01T18:58:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:58:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:43Z" name: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5.18be3f8090248103 namespace: kserve-ci-e2e-test resourceVersion: "20548" uid: ec224d4a-4537-4e34-aacb-860c35ad633e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5 namespace: kserve-ci-e2e-test resourceVersion: "19653" uid: d26e74b3-f903-4028-bf9f-ec2bb2dea9a8 kind: Event lastTimestamp: "2026-07-01T18:58:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:58:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:43Z" name: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5.18be3f80902848e6 namespace: kserve-ci-e2e-test resourceVersion: "20554" uid: f2f38d28-ae94-417f-85da-348133c568a8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5 namespace: kserve-ci-e2e-test resourceVersion: "19653" uid: d26e74b3-f903-4028-bf9f-ec2bb2dea9a8 kind: Event lastTimestamp: "2026-07-01T18:58:46Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:58:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:46Z" name: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5.18be3f811600ad02 namespace: kserve-ci-e2e-test resourceVersion: "20645" uid: 077a7a8c-1e57-4f6b-9727-0beddd9fb7eb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c059b-predictor-7c8bc8775f namespace: kserve-ci-e2e-test resourceVersion: "19629" uid: 666de3c5-97ae-460a-9381-93a1c1e3ec17 kind: Event lastTimestamp: "2026-07-01T18:57:33Z" message: 'Created pod: success-200-isvc-c059b-predictor-7c8bc8775f-9qkp5' metadata: creationTimestamp: "2026-07-01T18:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:57:33Z" name: success-200-isvc-c059b-predictor-7c8bc8775f.18be3f7034c4f8b0 namespace: kserve-ci-e2e-test resourceVersion: "19652" uid: 0c244488-3115-4c99-bfc5-073537953a19 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c059b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19627" uid: 5645b674-4dd2-47c0-98e7-f88b2846e7ce kind: Event lastTimestamp: "2026-07-01T18:57:33Z" message: Scaled up replica set success-200-isvc-c059b-predictor-7c8bc8775f from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:57:33Z" name: success-200-isvc-c059b-predictor.18be3f7030e859e1 namespace: kserve-ci-e2e-test resourceVersion: "19630" uid: 11313f74-a556-411d-9823-2e55f4ee00c2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:57:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c059b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19628" uid: 32d40aa5-7486-4478-82b9-c6a25e829a23 kind: Event lastTimestamp: "2026-07-01T18:58:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:57:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:58:18Z" name: success-200-isvc-c059b-predictor.18be3f73af739c86 namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: b4db494f-f11b-4b7d-9dd1-7d7d08d92283 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:57:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c059b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19628" uid: 32d40aa5-7486-4478-82b9-c6a25e829a23 kind: Event lastTimestamp: "2026-07-01T18:58:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:57:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:58:18Z" name: success-200-isvc-c059b-predictor.18be3f73aff183a8 namespace: kserve-ci-e2e-test resourceVersion: "20091" uid: f12ed2c3-f24d-454b-83aa-7e507405fff1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c059b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19862" uid: 32d40aa5-7486-4478-82b9-c6a25e829a23 kind: Event lastTimestamp: "2026-07-01T18:58:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:58:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:58:33Z" name: success-200-isvc-c059b-predictor.18be3f7e2c4cf594 namespace: kserve-ci-e2e-test resourceVersion: "20364" uid: 8ec0980d-cafa-42d9-8c9b-1a4f4d321f3b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c059b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19862" uid: 32d40aa5-7486-4478-82b9-c6a25e829a23 kind: Event lastTimestamp: "2026-07-01T18:58:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:58:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:58:33Z" name: success-200-isvc-c059b-predictor.18be3f7e2cc97636 namespace: kserve-ci-e2e-test resourceVersion: "20366" uid: a32a9cb2-04d6-4924-b322-b70aa36531dc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:57:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c059b namespace: kserve-ci-e2e-test resourceVersion: "19619" uid: f9258ec9-afb7-4f97-8b20-89790229ef67 kind: Event lastTimestamp: "2026-07-01T18:58:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:58:21Z" name: success-200-isvc-c059b.18be3f7030e77383 namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: d49b5a21-f750-435f-a174-9348cc192023 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c059b namespace: kserve-ci-e2e-test resourceVersion: "19619" uid: f9258ec9-afb7-4f97-8b20-89790229ef67 kind: Event lastTimestamp: "2026-07-01T18:57:33Z" message: 'Failed to update status for InferenceService "success-200-isvc-c059b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c059b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:57:33Z" name: success-200-isvc-c059b.18be3f70316f938f namespace: kserve-ci-e2e-test resourceVersion: "19633" uid: ed11f450-4a31-4b5b-90f5-b5b1b2239537 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c059b namespace: kserve-ci-e2e-test resourceVersion: "19619" uid: f9258ec9-afb7-4f97-8b20-89790229ef67 kind: Event lastTimestamp: "2026-07-01T18:57:33Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c059b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:57:33Z" name: success-200-isvc-c059b.18be3f70316fc7ef namespace: kserve-ci-e2e-test resourceVersion: "19635" uid: 87655eb7-a9be-44a4-860d-84057a80433d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c059b namespace: kserve-ci-e2e-test resourceVersion: "20116" uid: f9258ec9-afb7-4f97-8b20-89790229ef67 kind: Event lastTimestamp: "2026-07-01T18:58:21Z" message: InferenceService [success-200-isvc-c059b] is Ready metadata: creationTimestamp: "2026-07-01T18:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:58:21Z" name: success-200-isvc-c059b.18be3f7b4a0c893d namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: a9443584-ae54-424a-995d-9205e51bf23b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:59:53.805771Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c3262-predictor-865fd8b765-p828d namespace: kserve-ci-e2e-test resourceVersion: "21380" uid: 440a82cc-2ac4-4b80-ac8d-075af9ca1394 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c3262-predictor-865fd8b765-p828d to ip-10-0-129-245.ec2.internal metadata: creationTimestamp: "2026-07-01T18:59:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:59:53Z" name: success-200-isvc-c3262-predictor-865fd8b765-p828d.18be3f90d7548d52 namespace: kserve-ci-e2e-test resourceVersion: "21385" uid: a64b9c97-419f-4288-820a-17be0975ecef reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:54Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c3262-predictor-865fd8b765-p828d namespace: kserve-ci-e2e-test resourceVersion: "21381" uid: 440a82cc-2ac4-4b80-ac8d-075af9ca1394 kind: Event lastTimestamp: "2026-07-01T18:59:54Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-c3262-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:54Z" name: success-200-isvc-c3262-predictor-865fd8b765-p828d.18be3f90e4e080db namespace: kserve-ci-e2e-test resourceVersion: "21437" uid: 6e4414ef-3337-4de9-8618-5f2b6bd1b81e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:54Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c3262-predictor-865fd8b765-p828d namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: 440a82cc-2ac4-4b80-ac8d-075af9ca1394 kind: Event lastTimestamp: "2026-07-01T18:59:54Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:59:54Z" name: success-200-isvc-c3262-predictor-865fd8b765-p828d.18be3f911560de9b namespace: kserve-ci-e2e-test resourceVersion: "21461" uid: cfa08b00-67d0-485e-a868-19ced7e810a4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3262-predictor-865fd8b765-p828d namespace: kserve-ci-e2e-test resourceVersion: "21381" uid: 440a82cc-2ac4-4b80-ac8d-075af9ca1394 kind: Event lastTimestamp: "2026-07-01T18:59:54Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T18:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:54Z" name: success-200-isvc-c3262-predictor-865fd8b765-p828d.18be3f9116c9bb27 namespace: kserve-ci-e2e-test resourceVersion: "21463" uid: 6043c4af-db60-46cc-ab58-067d96fb3655 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3262-predictor-865fd8b765-p828d namespace: kserve-ci-e2e-test resourceVersion: "21381" uid: 440a82cc-2ac4-4b80-ac8d-075af9ca1394 kind: Event lastTimestamp: "2026-07-01T18:59:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:54Z" name: success-200-isvc-c3262-predictor-865fd8b765-p828d.18be3f9119692b6c namespace: kserve-ci-e2e-test resourceVersion: "21465" uid: bf4ab740-5544-4750-8442-71bd481c7121 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3262-predictor-865fd8b765-p828d namespace: kserve-ci-e2e-test resourceVersion: "21381" uid: 440a82cc-2ac4-4b80-ac8d-075af9ca1394 kind: Event lastTimestamp: "2026-07-01T18:59:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:54Z" name: success-200-isvc-c3262-predictor-865fd8b765-p828d.18be3f9119d4291e namespace: kserve-ci-e2e-test resourceVersion: "21466" uid: 5b1b990e-a8a2-4fe8-8078-5a8f7d518442 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3262-predictor-865fd8b765-p828d namespace: kserve-ci-e2e-test resourceVersion: "21381" uid: 440a82cc-2ac4-4b80-ac8d-075af9ca1394 kind: Event lastTimestamp: "2026-07-01T18:59:54Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:54Z" name: success-200-isvc-c3262-predictor-865fd8b765-p828d.18be3f9119dd3b28 namespace: kserve-ci-e2e-test resourceVersion: "21467" uid: 2f9cbf80-9854-4420-981b-cbb4ce10a8db reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3262-predictor-865fd8b765-p828d namespace: kserve-ci-e2e-test resourceVersion: "21381" uid: 440a82cc-2ac4-4b80-ac8d-075af9ca1394 kind: Event lastTimestamp: "2026-07-01T18:59:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:54Z" name: success-200-isvc-c3262-predictor-865fd8b765-p828d.18be3f911d564f3c namespace: kserve-ci-e2e-test resourceVersion: "21469" uid: 574d108f-9f8e-4c60-a767-c1746d54efad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3262-predictor-865fd8b765-p828d namespace: kserve-ci-e2e-test resourceVersion: "21381" uid: 440a82cc-2ac4-4b80-ac8d-075af9ca1394 kind: Event lastTimestamp: "2026-07-01T18:59:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:59:54Z" name: success-200-isvc-c3262-predictor-865fd8b765-p828d.18be3f911dc58f43 namespace: kserve-ci-e2e-test resourceVersion: "21470" uid: 63abd65c-631b-489e-a31c-16e069a8e7f5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T18:59:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3262-predictor-865fd8b765-p828d namespace: kserve-ci-e2e-test resourceVersion: "21381" uid: 440a82cc-2ac4-4b80-ac8d-075af9ca1394 kind: Event lastTimestamp: "2026-07-01T19:00:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:59:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:00:32Z" name: success-200-isvc-c3262-predictor-865fd8b765-p828d.18be3f914b4035f1 namespace: kserve-ci-e2e-test resourceVersion: "21811" uid: 82992c9f-5f10-41d4-9761-2e1a1bf47222 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3262-predictor-865fd8b765-p828d namespace: kserve-ci-e2e-test resourceVersion: "21381" uid: 440a82cc-2ac4-4b80-ac8d-075af9ca1394 kind: Event lastTimestamp: "2026-07-01T19:09:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T19:09:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:08Z" name: success-200-isvc-c3262-predictor-865fd8b765-p828d.18be401203fd02db namespace: kserve-ci-e2e-test resourceVersion: "25961" uid: b3bacef2-2290-4e35-948c-61f82bfe4542 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:09:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3262-predictor-865fd8b765-p828d namespace: kserve-ci-e2e-test resourceVersion: "21381" uid: 440a82cc-2ac4-4b80-ac8d-075af9ca1394 kind: Event lastTimestamp: "2026-07-01T19:09:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T19:09:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:09:08Z" name: success-200-isvc-c3262-predictor-865fd8b765-p828d.18be401203fe17d3 namespace: kserve-ci-e2e-test resourceVersion: "25962" uid: 5db5e6ec-51a5-4159-a019-c27696d68092 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c3262-predictor-865fd8b765 namespace: kserve-ci-e2e-test resourceVersion: "21373" uid: 9d172387-3e06-4bdb-9ddc-53dc8447e654 kind: Event lastTimestamp: "2026-07-01T18:59:53Z" message: 'Created pod: success-200-isvc-c3262-predictor-865fd8b765-p828d' metadata: creationTimestamp: "2026-07-01T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:59:53Z" name: success-200-isvc-c3262-predictor-865fd8b765.18be3f90d6e7fda1 namespace: kserve-ci-e2e-test resourceVersion: "21382" uid: b09266ff-8819-4ccd-8690-ece2f1dda25d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c3262-predictor namespace: kserve-ci-e2e-test resourceVersion: "21369" uid: b14aa2c5-c81c-46b7-8a91-6cf840f88920 kind: Event lastTimestamp: "2026-07-01T18:59:53Z" message: Scaled up replica set success-200-isvc-c3262-predictor-865fd8b765 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:59:53Z" name: success-200-isvc-c3262-predictor.18be3f90d4e6cce7 namespace: kserve-ci-e2e-test resourceVersion: "21374" uid: e767a770-6ae6-4a0d-906a-92af7af58b6a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:00:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3262-predictor namespace: kserve-ci-e2e-test resourceVersion: "21372" uid: ffa1bc95-5961-4845-bd24-3c0b7f3cc33f kind: Event lastTimestamp: "2026-07-01T19:00:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:00:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:00:23Z" name: success-200-isvc-c3262-predictor.18be3f94538b125b namespace: kserve-ci-e2e-test resourceVersion: "21739" uid: 04cb0e44-8eba-4703-8737-dea43b0344a3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T19:00:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3262-predictor namespace: kserve-ci-e2e-test resourceVersion: "21372" uid: ffa1bc95-5961-4845-bd24-3c0b7f3cc33f kind: Event lastTimestamp: "2026-07-01T19:00:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:00:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:00:23Z" name: success-200-isvc-c3262-predictor.18be3f9454016cc8 namespace: kserve-ci-e2e-test resourceVersion: "21741" uid: 2fdf36f1-f149-4451-88fe-4f5c6e9cf529 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:00:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3262-predictor namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: ffa1bc95-5961-4845-bd24-3c0b7f3cc33f kind: Event lastTimestamp: "2026-07-01T19:01:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:00:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:01:23Z" name: success-200-isvc-c3262-predictor.18be3f9b51a59d36 namespace: kserve-ci-e2e-test resourceVersion: "22229" uid: 8f91e7ab-6582-41fa-9187-24444e22e2d9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T19:00:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3262-predictor namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: ffa1bc95-5961-4845-bd24-3c0b7f3cc33f kind: Event lastTimestamp: "2026-07-01T19:01:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:00:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:01:23Z" name: success-200-isvc-c3262-predictor.18be3f9b5206ec92 namespace: kserve-ci-e2e-test resourceVersion: "22230" uid: 73fd96d5-3345-4f7c-9307-bb7efe7613ca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:59:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3262 namespace: kserve-ci-e2e-test resourceVersion: "21347" uid: b84b1b7f-4833-4193-a5d4-e5c7c92d3599 kind: Event lastTimestamp: "2026-07-01T19:00:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:00:42Z" name: success-200-isvc-c3262.18be3f90d500290a namespace: kserve-ci-e2e-test resourceVersion: "21899" uid: 9e5310e3-c53b-4e7a-83bc-17a0954d637b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3262 namespace: kserve-ci-e2e-test resourceVersion: "21347" uid: b84b1b7f-4833-4193-a5d4-e5c7c92d3599 kind: Event lastTimestamp: "2026-07-01T18:59:53Z" message: 'Failed to update status for InferenceService "success-200-isvc-c3262": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c3262": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:59:53Z" name: success-200-isvc-c3262.18be3f90d5870e19 namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: 4b541dbe-1026-45e1-bdb8-2c800f90b6d9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:59:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3262 namespace: kserve-ci-e2e-test resourceVersion: "21347" uid: b84b1b7f-4833-4193-a5d4-e5c7c92d3599 kind: Event lastTimestamp: "2026-07-01T18:59:53Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c3262": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T18:59:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:59:53Z" name: success-200-isvc-c3262.18be3f90d58764d3 namespace: kserve-ci-e2e-test resourceVersion: "21378" uid: 10f572c2-91d0-4ccf-8ff2-c4c207da82b9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:00:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3262 namespace: kserve-ci-e2e-test resourceVersion: "21896" uid: b84b1b7f-4833-4193-a5d4-e5c7c92d3599 kind: Event lastTimestamp: "2026-07-01T19:00:42Z" message: InferenceService [success-200-isvc-c3262] is Ready metadata: creationTimestamp: "2026-07-01T19:00:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:00:42Z" name: success-200-isvc-c3262.18be3f9c4375068d namespace: kserve-ci-e2e-test resourceVersion: "21898" uid: 71813f30-5e0a-4d92-b711-1f8f1da60964 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:49:14.688613Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-eb211-predictor-69b49c778c-qn8vg namespace: kserve-ci-e2e-test resourceVersion: "15917" uid: 1f74764e-8c3c-4486-a25f-1a5c2126aff6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-eb211-predictor-69b49c778c-qn8vg to ip-10-0-129-245.ec2.internal metadata: creationTimestamp: "2026-07-01T18:49:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:49:14Z" name: success-200-isvc-eb211-predictor-69b49c778c-qn8vg.18be3efc08faa8b2 namespace: kserve-ci-e2e-test resourceVersion: "15926" uid: 20434669-4b0d-4a18-985c-ffb1ffc47f98 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:14Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-eb211-predictor-69b49c778c-qn8vg namespace: kserve-ci-e2e-test resourceVersion: "15923" uid: 1f74764e-8c3c-4486-a25f-1a5c2126aff6 kind: Event lastTimestamp: "2026-07-01T18:49:14Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-eb211-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T18:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:14Z" name: success-200-isvc-eb211-predictor-69b49c778c-qn8vg.18be3efc107d512d namespace: kserve-ci-e2e-test resourceVersion: "15959" uid: 44e7d8b5-bb65-4df4-9a87-78bcb3f30048 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:15Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-eb211-predictor-69b49c778c-qn8vg namespace: kserve-ci-e2e-test resourceVersion: "15935" uid: 1f74764e-8c3c-4486-a25f-1a5c2126aff6 kind: Event lastTimestamp: "2026-07-01T18:49:15Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:49:15Z" name: success-200-isvc-eb211-predictor-69b49c778c-qn8vg.18be3efc4627fed0 namespace: kserve-ci-e2e-test resourceVersion: "15996" uid: 73133726-995d-4e39-bac0-a9b7251a2ef8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-eb211-predictor-69b49c778c-qn8vg namespace: kserve-ci-e2e-test resourceVersion: "15923" uid: 1f74764e-8c3c-4486-a25f-1a5c2126aff6 kind: Event lastTimestamp: "2026-07-01T18:49:15Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T18:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:15Z" name: success-200-isvc-eb211-predictor-69b49c778c-qn8vg.18be3efc47781d69 namespace: kserve-ci-e2e-test resourceVersion: "15998" uid: b92368ed-b556-4c3b-93fe-41a3ec36a38e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-eb211-predictor-69b49c778c-qn8vg namespace: kserve-ci-e2e-test resourceVersion: "15923" uid: 1f74764e-8c3c-4486-a25f-1a5c2126aff6 kind: Event lastTimestamp: "2026-07-01T18:49:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:15Z" name: success-200-isvc-eb211-predictor-69b49c778c-qn8vg.18be3efc4a9ad86f namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: ac548425-ed6a-40f0-a33a-8f145b1ab725 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-eb211-predictor-69b49c778c-qn8vg namespace: kserve-ci-e2e-test resourceVersion: "15923" uid: 1f74764e-8c3c-4486-a25f-1a5c2126aff6 kind: Event lastTimestamp: "2026-07-01T18:49:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:15Z" name: success-200-isvc-eb211-predictor-69b49c778c-qn8vg.18be3efc4b15effb namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: 6b4f85e9-ec78-489e-a98d-e55c27edcb3c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-eb211-predictor-69b49c778c-qn8vg namespace: kserve-ci-e2e-test resourceVersion: "15923" uid: 1f74764e-8c3c-4486-a25f-1a5c2126aff6 kind: Event lastTimestamp: "2026-07-01T18:49:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T18:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:15Z" name: success-200-isvc-eb211-predictor-69b49c778c-qn8vg.18be3efc4b20a87c namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 902ffebf-9b79-4117-bb5e-a745d1e2aecc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-eb211-predictor-69b49c778c-qn8vg namespace: kserve-ci-e2e-test resourceVersion: "15923" uid: 1f74764e-8c3c-4486-a25f-1a5c2126aff6 kind: Event lastTimestamp: "2026-07-01T18:49:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:15Z" name: success-200-isvc-eb211-predictor-69b49c778c-qn8vg.18be3efc4f0d2553 namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: f0b5ebc3-c90e-4f96-a2a0-a8ac1f105140 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-eb211-predictor-69b49c778c-qn8vg namespace: kserve-ci-e2e-test resourceVersion: "15923" uid: 1f74764e-8c3c-4486-a25f-1a5c2126aff6 kind: Event lastTimestamp: "2026-07-01T18:49:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:15Z" name: success-200-isvc-eb211-predictor-69b49c778c-qn8vg.18be3efc4f83d850 namespace: kserve-ci-e2e-test resourceVersion: "16005" uid: 5e21ded6-29d4-405e-b20e-8b46a91b2c2d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T18:49:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-eb211-predictor-69b49c778c-qn8vg namespace: kserve-ci-e2e-test resourceVersion: "15923" uid: 1f74764e-8c3c-4486-a25f-1a5c2126aff6 kind: Event lastTimestamp: "2026-07-01T18:58:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:32Z" name: success-200-isvc-eb211-predictor-69b49c778c-qn8vg.18be3efc67a4eccc namespace: kserve-ci-e2e-test resourceVersion: "20344" uid: cf803ebd-6a20-4cce-b4ba-576d0474e484 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-eb211-predictor-69b49c778c-qn8vg namespace: kserve-ci-e2e-test resourceVersion: "15923" uid: 1f74764e-8c3c-4486-a25f-1a5c2126aff6 kind: Event lastTimestamp: "2026-07-01T18:58:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:29Z" name: success-200-isvc-eb211-predictor-69b49c778c-qn8vg.18be3f7d3366d06a namespace: kserve-ci-e2e-test resourceVersion: "20246" uid: d91fdb79-d99e-465c-94e8-d534710c331f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-eb211-predictor-69b49c778c-qn8vg namespace: kserve-ci-e2e-test resourceVersion: "15923" uid: 1f74764e-8c3c-4486-a25f-1a5c2126aff6 kind: Event lastTimestamp: "2026-07-01T18:58:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:58:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:29Z" name: success-200-isvc-eb211-predictor-69b49c778c-qn8vg.18be3f7d33671920 namespace: kserve-ci-e2e-test resourceVersion: "20248" uid: e2a1280b-583c-478d-9af0-6a2e0ec70fa6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:58:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-eb211-predictor-69b49c778c-qn8vg namespace: kserve-ci-e2e-test resourceVersion: "15923" uid: 1f74764e-8c3c-4486-a25f-1a5c2126aff6 kind: Event lastTimestamp: "2026-07-01T18:58:32Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:58:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:32Z" name: success-200-isvc-eb211-predictor-69b49c778c-qn8vg.18be3f7ddc15e3b5 namespace: kserve-ci-e2e-test resourceVersion: "20342" uid: c727735f-9e87-43a2-a7a7-5fccf5dfa03d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-eb211-predictor-69b49c778c namespace: kserve-ci-e2e-test resourceVersion: "15901" uid: 1a442774-9e66-4573-8393-ea4a65cf3e22 kind: Event lastTimestamp: "2026-07-01T18:49:14Z" message: 'Created pod: success-200-isvc-eb211-predictor-69b49c778c-qn8vg' metadata: creationTimestamp: "2026-07-01T18:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:49:14Z" name: success-200-isvc-eb211-predictor-69b49c778c.18be3efc086570c5 namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: 7ee8c37e-a7ee-4043-b132-828ae025b285 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-eb211-predictor namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: d4898cc5-fc77-46c8-af9a-e8d68711c1aa kind: Event lastTimestamp: "2026-07-01T18:49:14Z" message: Scaled up replica set success-200-isvc-eb211-predictor-69b49c778c from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:49:14Z" name: success-200-isvc-eb211-predictor.18be3efc064309ed namespace: kserve-ci-e2e-test resourceVersion: "15904" uid: 79a34b8d-8350-4d4f-b388-d7e3667d6538 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:49:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-eb211-predictor namespace: kserve-ci-e2e-test resourceVersion: "15900" uid: baec6208-68eb-4dcb-9fc8-aabf1591ad7d kind: Event lastTimestamp: "2026-07-01T18:49:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:49:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:49:44Z" name: success-200-isvc-eb211-predictor.18be3eff859af6dd namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: 31a8e1a9-cc95-4713-87b4-d6d05cebc406 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T18:49:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-eb211-predictor namespace: kserve-ci-e2e-test resourceVersion: "15900" uid: baec6208-68eb-4dcb-9fc8-aabf1591ad7d kind: Event lastTimestamp: "2026-07-01T18:49:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:49:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:49:44Z" name: success-200-isvc-eb211-predictor.18be3eff860929e7 namespace: kserve-ci-e2e-test resourceVersion: "16306" uid: 99729b74-996f-456b-ad65-65df14de1175 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:49:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-eb211-predictor namespace: kserve-ci-e2e-test resourceVersion: "16187" uid: baec6208-68eb-4dcb-9fc8-aabf1591ad7d kind: Event lastTimestamp: "2026-07-01T18:50:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:49:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:50:44Z" name: success-200-isvc-eb211-predictor.18be3f06846cedcd namespace: kserve-ci-e2e-test resourceVersion: "16805" uid: 8fbafadb-7675-47c0-b7b1-a674399e02b2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T18:49:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-eb211-predictor namespace: kserve-ci-e2e-test resourceVersion: "16187" uid: baec6208-68eb-4dcb-9fc8-aabf1591ad7d kind: Event lastTimestamp: "2026-07-01T18:50:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:49:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:50:44Z" name: success-200-isvc-eb211-predictor.18be3f0684e2f485 namespace: kserve-ci-e2e-test resourceVersion: "16806" uid: 687e27f0-7053-4aa4-9f49-e6c2d163f518 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:49:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-eb211 namespace: kserve-ci-e2e-test resourceVersion: "15879" uid: 5b493ce3-c53f-4b0d-8ca0-aaf8062f57ad kind: Event lastTimestamp: "2026-07-01T18:50:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:50:02Z" name: success-200-isvc-eb211.18be3efc0682ba27 namespace: kserve-ci-e2e-test resourceVersion: "16465" uid: 4270fcc5-6c72-4883-9d0a-8922dde34669 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:50:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-eb211 namespace: kserve-ci-e2e-test resourceVersion: "16461" uid: 5b493ce3-c53f-4b0d-8ca0-aaf8062f57ad kind: Event lastTimestamp: "2026-07-01T18:50:02Z" message: InferenceService [success-200-isvc-eb211] is Ready metadata: creationTimestamp: "2026-07-01T18:50:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:50:02Z" name: success-200-isvc-eb211.18be3f0726d76f6e namespace: kserve-ci-e2e-test resourceVersion: "16462" uid: d49b4f70-a8b8-4d49-bd0b-aedb72845cc3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:46:44.755470Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4 namespace: kserve-ci-e2e-test resourceVersion: "14160" uid: a903f2e3-d219-46b2-ad31-5a1d31733461 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4 to ip-10-0-129-245.ec2.internal metadata: creationTimestamp: "2026-07-01T18:46:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:46:44Z" name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4.18be3ed920447549 namespace: kserve-ci-e2e-test resourceVersion: "14165" uid: 9a1ad6bd-fce4-4810-ad91-83ff1a6a2e13 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:45Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4 namespace: kserve-ci-e2e-test resourceVersion: "14170" uid: a903f2e3-d219-46b2-ad31-5a1d31733461 kind: Event lastTimestamp: "2026-07-01T18:46:45Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:46:45Z" name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4.18be3ed93ab1f310 namespace: kserve-ci-e2e-test resourceVersion: "14233" uid: 0d4582d5-a531-4ea6-a1b4-54bb41d7d3ef reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: a903f2e3-d219-46b2-ad31-5a1d31733461 kind: Event lastTimestamp: "2026-07-01T18:46:45Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" metadata: creationTimestamp: "2026-07-01T18:46:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:46:45Z" name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4.18be3ed93c416292 namespace: kserve-ci-e2e-test resourceVersion: "14240" uid: 6fec25b5-f10f-421c-9b92-491e0ad924c1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: a903f2e3-d219-46b2-ad31-5a1d31733461 kind: Event lastTimestamp: "2026-07-01T18:47:01Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1690" in 16.651s (16.651s including waiting). Image size: 1335614578 bytes.' metadata: creationTimestamp: "2026-07-01T18:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:01Z" name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4.18be3edd1cc70048 namespace: kserve-ci-e2e-test resourceVersion: "14460" uid: a31ce29c-6ffa-4e82-a233-c459e26b6023 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: a903f2e3-d219-46b2-ad31-5a1d31733461 kind: Event lastTimestamp: "2026-07-01T18:47:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T18:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:01Z" name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4.18be3edd207e9a13 namespace: kserve-ci-e2e-test resourceVersion: "14461" uid: 65105449-931f-4653-8b86-35f73ee1d89f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: a903f2e3-d219-46b2-ad31-5a1d31733461 kind: Event lastTimestamp: "2026-07-01T18:47:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T18:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:01Z" name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4.18be3edd2117b502 namespace: kserve-ci-e2e-test resourceVersion: "14464" uid: eadc3169-8f53-490e-9cc8-1283f9b86683 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: a903f2e3-d219-46b2-ad31-5a1d31733461 kind: Event lastTimestamp: "2026-07-01T18:47:01Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T18:47:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:01Z" name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4.18be3edd212680cf namespace: kserve-ci-e2e-test resourceVersion: "14465" uid: 721479df-eb33-4ad5-8c7e-7f796b0fd5be reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: a903f2e3-d219-46b2-ad31-5a1d31733461 kind: Event lastTimestamp: "2026-07-01T18:47:04Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.275s (2.275s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T18:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:04Z" name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4.18be3edda8c6e2bd namespace: kserve-ci-e2e-test resourceVersion: "14483" uid: 93a04315-8fb9-46a2-8294-10c9f15f4095 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: a903f2e3-d219-46b2-ad31-5a1d31733461 kind: Event lastTimestamp: "2026-07-01T18:47:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T18:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:04Z" name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4.18be3eddac69f416 namespace: kserve-ci-e2e-test resourceVersion: "14485" uid: 6bbcfa02-e040-4f4f-bb94-1f79f84c010f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: a903f2e3-d219-46b2-ad31-5a1d31733461 kind: Event lastTimestamp: "2026-07-01T18:47:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:47:04Z" name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4.18be3eddace6fe53 namespace: kserve-ci-e2e-test resourceVersion: "14487" uid: 5ec52b77-b1d0-43f9-843d-0061669ee107 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T18:47:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: a903f2e3-d219-46b2-ad31-5a1d31733461 kind: Event lastTimestamp: "2026-07-01T18:48:20Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:20Z" name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4.18be3eddc90039ef namespace: kserve-ci-e2e-test resourceVersion: "15339" uid: d6662256-d881-43a0-8650-50fede6d0263 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: a903f2e3-d219-46b2-ad31-5a1d31733461 kind: Event lastTimestamp: "2026-07-01T18:48:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T18:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:18Z" name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4.18be3eef08055b05 namespace: kserve-ci-e2e-test resourceVersion: "15253" uid: 4a5be3ea-61bf-4685-9ac7-f4251c62ba65 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: a903f2e3-d219-46b2-ad31-5a1d31733461 kind: Event lastTimestamp: "2026-07-01T18:48:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T18:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:18Z" name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4.18be3eef0805b2da namespace: kserve-ci-e2e-test resourceVersion: "15256" uid: f98fecaf-0deb-4c42-9b3f-6dbfa79beddf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: a903f2e3-d219-46b2-ad31-5a1d31733461 kind: Event lastTimestamp: "2026-07-01T18:48:20Z" message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": dial tcp 10.134.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T18:48:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:20Z" name: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4.18be3eef7b454535 namespace: kserve-ci-e2e-test resourceVersion: "15338" uid: 9731ffb6-3db5-4ed0-9cfc-a8b6a870737d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-245.ec2.internal source: component: kubelet host: ip-10-0-129-245.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f2cdf-predictor-64d5b74467 namespace: kserve-ci-e2e-test resourceVersion: "14152" uid: d967769a-6b6b-484f-8959-0a7e14fad76e kind: Event lastTimestamp: "2026-07-01T18:46:44Z" message: 'Created pod: success-200-isvc-f2cdf-predictor-64d5b74467-n5sd4' metadata: creationTimestamp: "2026-07-01T18:46:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:46:44Z" name: success-200-isvc-f2cdf-predictor-64d5b74467.18be3ed91fa7db30 namespace: kserve-ci-e2e-test resourceVersion: "14162" uid: fbaf9590-1699-4fee-a06e-a151f04c8602 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:46:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f2cdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "14150" uid: 474f1b76-37c2-4c11-a80b-cf4852a6d6c2 kind: Event lastTimestamp: "2026-07-01T18:46:44Z" message: Scaled up replica set success-200-isvc-f2cdf-predictor-64d5b74467 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:46:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:46:44Z" name: success-200-isvc-f2cdf-predictor.18be3ed91cf4e428 namespace: kserve-ci-e2e-test resourceVersion: "14154" uid: 66056750-cae6-4baa-9205-892936df4f44 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:46:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f2cdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "14151" uid: a5354683-0657-4441-84c7-cd6caa785853 kind: Event lastTimestamp: "2026-07-01T18:47:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:47:44Z" name: success-200-isvc-f2cdf-predictor.18be3edc9bbb7915 namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: c0cddbff-9d51-4971-a3d9-6c8c94dc2128 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:46:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f2cdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "14151" uid: a5354683-0657-4441-84c7-cd6caa785853 kind: Event lastTimestamp: "2026-07-01T18:47:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:46:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:47:44Z" name: success-200-isvc-f2cdf-predictor.18be3edc9c267fcf namespace: kserve-ci-e2e-test resourceVersion: "14845" uid: 805fdbaf-3645-481a-8525-b6d0427d70c4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:47:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f2cdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: a5354683-0657-4441-84c7-cd6caa785853 kind: Event lastTimestamp: "2026-07-01T18:48:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:48:14Z" name: success-200-isvc-f2cdf-predictor.18be3ee399daabd8 namespace: kserve-ci-e2e-test resourceVersion: "15147" uid: ec2ac5ef-f580-41ed-8919-5bd9216447aa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T18:47:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f2cdf-predictor namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: a5354683-0657-4441-84c7-cd6caa785853 kind: Event lastTimestamp: "2026-07-01T18:48:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:48:14Z" name: success-200-isvc-f2cdf-predictor.18be3ee39a4f3f04 namespace: kserve-ci-e2e-test resourceVersion: "15148" uid: ab985000-5986-4e72-8c30-688f3093f90c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T18:46:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f2cdf namespace: kserve-ci-e2e-test resourceVersion: "14143" uid: fcc31b1d-254a-40cd-998f-d0906977a282 kind: Event lastTimestamp: "2026-07-01T18:47:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T18:46:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:47:50Z" name: success-200-isvc-f2cdf.18be3ed91cfad094 namespace: kserve-ci-e2e-test resourceVersion: "14903" uid: 40bbf1de-330e-4bde-a6fc-3879011c33c7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:47:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f2cdf namespace: kserve-ci-e2e-test resourceVersion: "14901" uid: fcc31b1d-254a-40cd-998f-d0906977a282 kind: Event lastTimestamp: "2026-07-01T18:47:50Z" message: InferenceService [success-200-isvc-f2cdf] is Ready metadata: creationTimestamp: "2026-07-01T18:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:47:50Z" name: success-200-isvc-f2cdf.18be3ee88551a8dc namespace: kserve-ci-e2e-test resourceVersion: "14902" uid: d341182e-82d8-4493-b33b-7efd2726d571 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T19:10:18.714021Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-43223-7d58669c44-z74rq namespace: kserve-ci-e2e-test resourceVersion: "26862" uid: 405c3fc8-43e9-40cb-bdab-a8a575984d31 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-43223-7d58669c44-z74rq to ip-10-0-139-178.ec2.internal metadata: creationTimestamp: "2026-07-01T19:10:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T19:10:18Z" name: switch-graph-43223-7d58669c44-z74rq.18be402256c3c252 namespace: kserve-ci-e2e-test resourceVersion: "26867" uid: a90f32c2-4845-4c6d-9f37-66da28dc7432 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:10:18Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-43223-7d58669c44-z74rq namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: 405c3fc8-43e9-40cb-bdab-a8a575984d31 kind: Event lastTimestamp: "2026-07-01T19:10:18Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-43223-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T19:10:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:10:18Z" name: switch-graph-43223-7d58669c44-z74rq.18be40226421196d namespace: kserve-ci-e2e-test resourceVersion: "26879" uid: 9518a2a8-da10-45f4-84fd-df7afb49d1b2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:10:19Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-43223-7d58669c44-z74rq namespace: kserve-ci-e2e-test resourceVersion: "26871" uid: 405c3fc8-43e9-40cb-bdab-a8a575984d31 kind: Event lastTimestamp: "2026-07-01T19:10:19Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T19:10:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T19:10:19Z" name: switch-graph-43223-7d58669c44-z74rq.18be4022943d5077 namespace: kserve-ci-e2e-test resourceVersion: "26891" uid: 9be02601-82c3-459b-9875-ef4539a0415d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:10:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-43223} kind: Pod name: switch-graph-43223-7d58669c44-z74rq namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: 405c3fc8-43e9-40cb-bdab-a8a575984d31 kind: Event lastTimestamp: "2026-07-01T19:10:19Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a589cb7fd67a31c767ba2472b47f816cadffddbd6c2e2c8ddfd876f9a79d8448" already present on machine metadata: creationTimestamp: "2026-07-01T19:10:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:10:19Z" name: switch-graph-43223-7d58669c44-z74rq.18be402295980791 namespace: kserve-ci-e2e-test resourceVersion: "26893" uid: 976d853e-0ebc-446f-a316-3b435abe1487 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:10:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-43223} kind: Pod name: switch-graph-43223-7d58669c44-z74rq namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: 405c3fc8-43e9-40cb-bdab-a8a575984d31 kind: Event lastTimestamp: "2026-07-01T19:10:19Z" message: 'Created container: switch-graph-43223' metadata: creationTimestamp: "2026-07-01T19:10:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:10:19Z" name: switch-graph-43223-7d58669c44-z74rq.18be402298a3155f namespace: kserve-ci-e2e-test resourceVersion: "26894" uid: 03163700-0184-481e-aa56-b5645403b1e2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:10:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-43223} kind: Pod name: switch-graph-43223-7d58669c44-z74rq namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: 405c3fc8-43e9-40cb-bdab-a8a575984d31 kind: Event lastTimestamp: "2026-07-01T19:10:19Z" message: Started container switch-graph-43223 metadata: creationTimestamp: "2026-07-01T19:10:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:10:19Z" name: switch-graph-43223-7d58669c44-z74rq.18be40229918618b namespace: kserve-ci-e2e-test resourceVersion: "26895" uid: f1eecd19-f144-43cb-85de-7cb2737df6fe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:26:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-43223} kind: Pod name: switch-graph-43223-7d58669c44-z74rq namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: 405c3fc8-43e9-40cb-bdab-a8a575984d31 kind: Event lastTimestamp: "2026-07-01T19:26:38Z" message: Stopping container switch-graph-43223 metadata: creationTimestamp: "2026-07-01T19:26:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:26:38Z" name: switch-graph-43223-7d58669c44-z74rq.18be41065c387066 namespace: kserve-ci-e2e-test resourceVersion: "33677" uid: 8f67475c-27a8-4eb0-9446-c31c2cbe1707 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T19:26:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-43223} kind: Pod name: switch-graph-43223-7d58669c44-z74rq namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: 405c3fc8-43e9-40cb-bdab-a8a575984d31 kind: Event lastTimestamp: "2026-07-01T19:27:01Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T19:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T19:27:01Z" name: switch-graph-43223-7d58669c44-z74rq.18be410729988119 namespace: kserve-ci-e2e-test resourceVersion: "33922" uid: 571e0f12-ef2e-483d-9b16-6f835e91940b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:10:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-43223-7d58669c44 namespace: kserve-ci-e2e-test resourceVersion: "26859" uid: be3fdad2-95a6-448b-ba5d-70829cef63e7 kind: Event lastTimestamp: "2026-07-01T19:10:18Z" message: 'Created pod: switch-graph-43223-7d58669c44-z74rq' metadata: creationTimestamp: "2026-07-01T19:10:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:10:18Z" name: switch-graph-43223-7d58669c44.18be4022564f4081 namespace: kserve-ci-e2e-test resourceVersion: "26863" uid: c6c525f1-05e2-4efa-bcc4-b1c251e2a265 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:10:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-43223 namespace: kserve-ci-e2e-test resourceVersion: "26857" uid: 17e12307-acc9-4f3f-b889-dedef9356327 kind: Event lastTimestamp: "2026-07-01T19:10:18Z" message: Scaled up replica set switch-graph-43223-7d58669c44 from 0 to 1 metadata: creationTimestamp: "2026-07-01T19:10:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:10:18Z" name: switch-graph-43223.18be4022559ebdd0 namespace: kserve-ci-e2e-test resourceVersion: "26860" uid: a6119619-06c4-4acc-b566-41fccb6f37ff reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:10:18Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-43223 namespace: kserve-ci-e2e-test resourceVersion: "26873" uid: 57d718bb-8f69-48a1-8693-9a254b5672b7 kind: Event lastTimestamp: "2026-07-01T19:10:18Z" message: InferenceGraph [switch-graph-43223] is Ready metadata: creationTimestamp: "2026-07-01T19:10:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T19:10:18Z" name: switch-graph-43223.18be40225b06577c namespace: kserve-ci-e2e-test resourceVersion: "26875" uid: 097a3c40-fef4-4779-bb93-ceac5a99e2f9 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:10:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-43223 namespace: kserve-ci-e2e-test resourceVersion: "26858" uid: c4eae417-5028-47cc-a719-f39a22b678c8 kind: Event lastTimestamp: "2026-07-01T19:10:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:10:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:10:33Z" name: switch-graph-43223.18be4025d4e15ee9 namespace: kserve-ci-e2e-test resourceVersion: "27005" uid: 4c2e6146-c9f8-4fb8-9208-11891531fd07 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:10:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-43223 namespace: kserve-ci-e2e-test resourceVersion: "26858" uid: c4eae417-5028-47cc-a719-f39a22b678c8 kind: Event lastTimestamp: "2026-07-01T19:10:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T19:10:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:10:33Z" name: switch-graph-43223.18be4025d55d2d21 namespace: kserve-ci-e2e-test resourceVersion: "27007" uid: 2c141635-4ef3-46e9-b479-d99e19e1e9fc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:10:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-43223 namespace: kserve-ci-e2e-test resourceVersion: "27006" uid: c4eae417-5028-47cc-a719-f39a22b678c8 kind: Event lastTimestamp: "2026-07-01T19:10:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:10:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:10:48Z" name: switch-graph-43223.18be40295434caf0 namespace: kserve-ci-e2e-test resourceVersion: "27116" uid: fbd80d4c-6fb0-46bf-8c30-4a025bbfabb4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T19:10:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-43223 namespace: kserve-ci-e2e-test resourceVersion: "27006" uid: c4eae417-5028-47cc-a719-f39a22b678c8 kind: Event lastTimestamp: "2026-07-01T19:10:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T19:10:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T19:10:48Z" name: switch-graph-43223.18be4029549dfb29 namespace: kserve-ci-e2e-test resourceVersion: "27117" uid: 12bed63d-2d52-4107-824a-c05ef9037098 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:49:18.813558Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9857d-698d9d6767-gzxql namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: 0b236198-ea57-4094-814f-197d7b3adcf3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-9857d-698d9d6767-gzxql to ip-10-0-139-178.ec2.internal metadata: creationTimestamp: "2026-07-01T18:49:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:49:18Z" name: switch-graph-9857d-698d9d6767-gzxql.18be3efcfed85273 namespace: kserve-ci-e2e-test resourceVersion: "16068" uid: e4afb06f-3ae5-4e1b-9cab-7b7868109c06 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:19Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9857d-698d9d6767-gzxql namespace: kserve-ci-e2e-test resourceVersion: "16071" uid: 0b236198-ea57-4094-814f-197d7b3adcf3 kind: Event lastTimestamp: "2026-07-01T18:49:19Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:49:19Z" name: switch-graph-9857d-698d9d6767-gzxql.18be3efd184a05b2 namespace: kserve-ci-e2e-test resourceVersion: "16089" uid: 34a2e349-7912-4737-b0b9-a4772988dd3d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9857d} kind: Pod name: switch-graph-9857d-698d9d6767-gzxql namespace: kserve-ci-e2e-test resourceVersion: "16065" uid: 0b236198-ea57-4094-814f-197d7b3adcf3 kind: Event lastTimestamp: "2026-07-01T18:49:19Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:a589cb7fd67a31c767ba2472b47f816cadffddbd6c2e2c8ddfd876f9a79d8448" already present on machine metadata: creationTimestamp: "2026-07-01T18:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:19Z" name: switch-graph-9857d-698d9d6767-gzxql.18be3efd19a789af namespace: kserve-ci-e2e-test resourceVersion: "16091" uid: 6e4ca7a1-dd35-4cb4-b6d7-204efb367868 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9857d} kind: Pod name: switch-graph-9857d-698d9d6767-gzxql namespace: kserve-ci-e2e-test resourceVersion: "16065" uid: 0b236198-ea57-4094-814f-197d7b3adcf3 kind: Event lastTimestamp: "2026-07-01T18:49:19Z" message: 'Created container: switch-graph-9857d' metadata: creationTimestamp: "2026-07-01T18:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:19Z" name: switch-graph-9857d-698d9d6767-gzxql.18be3efd1d02c2de namespace: kserve-ci-e2e-test resourceVersion: "16092" uid: b8ef3382-f975-4dc9-aaa8-b7b584118a32 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9857d} kind: Pod name: switch-graph-9857d-698d9d6767-gzxql namespace: kserve-ci-e2e-test resourceVersion: "16065" uid: 0b236198-ea57-4094-814f-197d7b3adcf3 kind: Event lastTimestamp: "2026-07-01T18:49:19Z" message: Started container switch-graph-9857d metadata: creationTimestamp: "2026-07-01T18:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:49:19Z" name: switch-graph-9857d-698d9d6767-gzxql.18be3efd1d6490a5 namespace: kserve-ci-e2e-test resourceVersion: "16093" uid: a015aa76-1f5c-4f37-96c7-2457aa1b2bd0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:57:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9857d} kind: Pod name: switch-graph-9857d-698d9d6767-gzxql namespace: kserve-ci-e2e-test resourceVersion: "16065" uid: 0b236198-ea57-4094-814f-197d7b3adcf3 kind: Event lastTimestamp: "2026-07-01T18:57:33Z" message: Stopping container switch-graph-9857d metadata: creationTimestamp: "2026-07-01T18:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:57:33Z" name: switch-graph-9857d-698d9d6767-gzxql.18be3f7028e132b1 namespace: kserve-ci-e2e-test resourceVersion: "19614" uid: b794d4cd-c3da-4b8e-9e10-d1721e783df6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:57:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9857d} kind: Pod name: switch-graph-9857d-698d9d6767-gzxql namespace: kserve-ci-e2e-test resourceVersion: "16065" uid: 0b236198-ea57-4094-814f-197d7b3adcf3 kind: Event lastTimestamp: "2026-07-01T18:58:00Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T18:57:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:58:00Z" name: switch-graph-9857d-698d9d6767-gzxql.18be3f70bd85edd1 namespace: kserve-ci-e2e-test resourceVersion: "19954" uid: ca1e3cee-f956-4c18-a907-5ccc8a503d16 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-9857d-698d9d6767 namespace: kserve-ci-e2e-test resourceVersion: "16058" uid: f8471bbe-46f6-4c04-be22-27150b4e35cc kind: Event lastTimestamp: "2026-07-01T18:49:18Z" message: 'Created pod: switch-graph-9857d-698d9d6767-gzxql' metadata: creationTimestamp: "2026-07-01T18:49:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:49:18Z" name: switch-graph-9857d-698d9d6767.18be3efcfe66c1e5 namespace: kserve-ci-e2e-test resourceVersion: "16064" uid: 5b6a587f-5b52-42eb-b810-5e61da897f3a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-9857d namespace: kserve-ci-e2e-test resourceVersion: "16055" uid: 6e9b4757-7810-4daa-b6d8-c87525272ed9 kind: Event lastTimestamp: "2026-07-01T18:49:18Z" message: Scaled up replica set switch-graph-9857d-698d9d6767 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:49:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:49:18Z" name: switch-graph-9857d.18be3efcfda6062b namespace: kserve-ci-e2e-test resourceVersion: "16061" uid: 8c9aa6e9-e506-4e1c-a894-24a9e777a643 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:18Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-9857d namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: 187b9c55-ccf0-4a4b-ae1e-7bd7e013e127 kind: Event lastTimestamp: "2026-07-01T18:49:18Z" message: InferenceGraph [switch-graph-9857d] is Ready metadata: creationTimestamp: "2026-07-01T18:49:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:49:18Z" name: switch-graph-9857d.18be3efd03a5bb94 namespace: kserve-ci-e2e-test resourceVersion: "16076" uid: 1072ce3f-1b04-4633-8afa-344f58614e46 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-9857d namespace: kserve-ci-e2e-test resourceVersion: "16056" uid: 73e0a4f4-5506-430b-b778-48aa9a301bad kind: Event lastTimestamp: "2026-07-01T18:49:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:49:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:49:33Z" name: switch-graph-9857d.18be3f007c0984e0 namespace: kserve-ci-e2e-test resourceVersion: "16222" uid: 85681bd6-f0d5-47a9-a0f3-2fe3d9a097e5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-9857d namespace: kserve-ci-e2e-test resourceVersion: "16056" uid: 73e0a4f4-5506-430b-b778-48aa9a301bad kind: Event lastTimestamp: "2026-07-01T18:49:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T18:49:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:49:33Z" name: switch-graph-9857d.18be3f007c958631 namespace: kserve-ci-e2e-test resourceVersion: "16226" uid: c0743848-a097-4fa6-8c4c-6e4ad8bd8a4c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-9857d namespace: kserve-ci-e2e-test resourceVersion: "16223" uid: 73e0a4f4-5506-430b-b778-48aa9a301bad kind: Event lastTimestamp: "2026-07-01T18:49:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:49:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:49:48Z" name: switch-graph-9857d.18be3f03fb7dbae7 namespace: kserve-ci-e2e-test resourceVersion: "16340" uid: df60fe57-25c7-4364-8390-4ebfe65dae50 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:49:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-9857d namespace: kserve-ci-e2e-test resourceVersion: "16223" uid: 73e0a4f4-5506-430b-b778-48aa9a301bad kind: Event lastTimestamp: "2026-07-01T18:49:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T18:49:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:49:48Z" name: switch-graph-9857d.18be3f03fc1d7b57 namespace: kserve-ci-e2e-test resourceVersion: "16344" uid: 131b58c2-2bb7-4512-816f-c51c094471a0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T18:48:04.441318Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-f2cdf-545944cd54-5rxqp namespace: kserve-ci-e2e-test resourceVersion: "15035" uid: cef20b52-23bf-4711-8ff0-91fcc81a715e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-f2cdf-545944cd54-5rxqp to ip-10-0-139-178.ec2.internal metadata: creationTimestamp: "2026-07-01T18:48:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T18:48:04Z" name: switch-graph-f2cdf-545944cd54-5rxqp.18be3eebade9fdfb namespace: kserve-ci-e2e-test resourceVersion: "15040" uid: bb86829a-6aff-4d03-8dbe-6b1d101285ed reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-674966d9c7-h6mm8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:04Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-f2cdf-545944cd54-5rxqp namespace: kserve-ci-e2e-test resourceVersion: "15044" uid: cef20b52-23bf-4711-8ff0-91fcc81a715e kind: Event lastTimestamp: "2026-07-01T18:48:04Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T18:48:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T18:48:04Z" name: switch-graph-f2cdf-545944cd54-5rxqp.18be3eebc780321a namespace: kserve-ci-e2e-test resourceVersion: "15055" uid: 1d801117-b6fc-4e9f-af53-fc4929d7292f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f2cdf} kind: Pod name: switch-graph-f2cdf-545944cd54-5rxqp namespace: kserve-ci-e2e-test resourceVersion: "15037" uid: cef20b52-23bf-4711-8ff0-91fcc81a715e kind: Event lastTimestamp: "2026-07-01T18:48:04Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:a589cb7fd67a31c767ba2472b47f816cadffddbd6c2e2c8ddfd876f9a79d8448" metadata: creationTimestamp: "2026-07-01T18:48:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:04Z" name: switch-graph-f2cdf-545944cd54-5rxqp.18be3eebc9000f56 namespace: kserve-ci-e2e-test resourceVersion: "15057" uid: 9c45fbb4-183b-4759-975c-a5c1f534b5cb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f2cdf} kind: Pod name: switch-graph-f2cdf-545944cd54-5rxqp namespace: kserve-ci-e2e-test resourceVersion: "15037" uid: cef20b52-23bf-4711-8ff0-91fcc81a715e kind: Event lastTimestamp: "2026-07-01T18:48:07Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:a589cb7fd67a31c767ba2472b47f816cadffddbd6c2e2c8ddfd876f9a79d8448" in 3.079s (3.079s including waiting). Image size: 218070140 bytes.' metadata: creationTimestamp: "2026-07-01T18:48:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:07Z" name: switch-graph-f2cdf-545944cd54-5rxqp.18be3eec808e93d3 namespace: kserve-ci-e2e-test resourceVersion: "15083" uid: f7868051-0b34-4241-9736-e8a2064c71ae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f2cdf} kind: Pod name: switch-graph-f2cdf-545944cd54-5rxqp namespace: kserve-ci-e2e-test resourceVersion: "15037" uid: cef20b52-23bf-4711-8ff0-91fcc81a715e kind: Event lastTimestamp: "2026-07-01T18:48:08Z" message: 'Created container: switch-graph-f2cdf' metadata: creationTimestamp: "2026-07-01T18:48:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:08Z" name: switch-graph-f2cdf-545944cd54-5rxqp.18be3eec8456003c namespace: kserve-ci-e2e-test resourceVersion: "15084" uid: 7c3fefa8-18c0-4e5d-a94d-1bfb7a875baf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f2cdf} kind: Pod name: switch-graph-f2cdf-545944cd54-5rxqp namespace: kserve-ci-e2e-test resourceVersion: "15037" uid: cef20b52-23bf-4711-8ff0-91fcc81a715e kind: Event lastTimestamp: "2026-07-01T18:48:08Z" message: Started container switch-graph-f2cdf metadata: creationTimestamp: "2026-07-01T18:48:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:08Z" name: switch-graph-f2cdf-545944cd54-5rxqp.18be3eec84c985c2 namespace: kserve-ci-e2e-test resourceVersion: "15085" uid: 9366aa3f-728e-4184-8444-147103401462 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f2cdf} kind: Pod name: switch-graph-f2cdf-545944cd54-5rxqp namespace: kserve-ci-e2e-test resourceVersion: "15037" uid: cef20b52-23bf-4711-8ff0-91fcc81a715e kind: Event lastTimestamp: "2026-07-01T18:48:18Z" message: Stopping container switch-graph-f2cdf metadata: creationTimestamp: "2026-07-01T18:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:18Z" name: switch-graph-f2cdf-545944cd54-5rxqp.18be3eeef904fe06 namespace: kserve-ci-e2e-test resourceVersion: "15199" uid: 864a0267-14f6-4a33-b1e4-612300a9a776 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T18:48:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f2cdf} kind: Pod name: switch-graph-f2cdf-545944cd54-5rxqp namespace: kserve-ci-e2e-test resourceVersion: "15037" uid: cef20b52-23bf-4711-8ff0-91fcc81a715e kind: Event lastTimestamp: "2026-07-01T18:48:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T18:48:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T18:48:44Z" name: switch-graph-f2cdf-545944cd54-5rxqp.18be3eef35e031f2 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: f6ecd12e-7863-478d-a955-98b0c1c7de96 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-178.ec2.internal source: component: kubelet host: ip-10-0-139-178.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-f2cdf-545944cd54 namespace: kserve-ci-e2e-test resourceVersion: "15032" uid: 212e99a5-6ef8-4c80-a497-ea6ca8caeaeb kind: Event lastTimestamp: "2026-07-01T18:48:04Z" message: 'Created pod: switch-graph-f2cdf-545944cd54-5rxqp' metadata: creationTimestamp: "2026-07-01T18:48:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:48:04Z" name: switch-graph-f2cdf-545944cd54.18be3eebad787296 namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: bf6a6068-9ed9-43c7-9d44-60cd67e675b1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-f2cdf namespace: kserve-ci-e2e-test resourceVersion: "15030" uid: 482729b8-8def-422e-af34-916a2594ae1f kind: Event lastTimestamp: "2026-07-01T18:48:04Z" message: Scaled up replica set switch-graph-f2cdf-545944cd54 from 0 to 1 metadata: creationTimestamp: "2026-07-01T18:48:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T18:48:04Z" name: switch-graph-f2cdf.18be3eebac642fc8 namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: dafc57bb-fe5f-461c-bafa-908f761b055b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T18:48:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-f2cdf namespace: kserve-ci-e2e-test resourceVersion: "15046" uid: b1f420a8-ff93-47de-b478-6d9fa49a03dc kind: Event lastTimestamp: "2026-07-01T18:48:04Z" message: InferenceGraph [switch-graph-f2cdf] is Ready metadata: creationTimestamp: "2026-07-01T18:48:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T18:48:04Z" name: switch-graph-f2cdf.18be3eebb1263207 namespace: kserve-ci-e2e-test resourceVersion: "15048" uid: 99d683f0-e2ee-498e-a367-4ef19500d321 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal kind: EventList metadata: resourceVersion: "33951"