--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:38:34.770117Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-57d01-54cddf9746-zqjch namespace: kserve-ci-e2e-test resourceVersion: "20987" uid: c1fee85a-ce18-47e7-8f0a-74efdf5004d7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-57d01-54cddf9746-zqjch to ip-10-0-143-249.ec2.internal metadata: creationTimestamp: "2026-07-02T14:38:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:38:34Z" name: ensemble-graph-57d01-54cddf9746-zqjch.18be7fe2d949cb4b namespace: kserve-ci-e2e-test resourceVersion: "20992" uid: f03d0291-b077-4b14-9d55-6422d3077eb5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:35Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-57d01-54cddf9746-zqjch namespace: kserve-ci-e2e-test resourceVersion: "20996" uid: c1fee85a-ce18-47e7-8f0a-74efdf5004d7 kind: Event lastTimestamp: "2026-07-02T14:38:35Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:38:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:38:35Z" name: ensemble-graph-57d01-54cddf9746-zqjch.18be7fe2f3100593 namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: f5f60f5c-11d5-476d-b127-10763bea26b4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-57d01} kind: Pod name: ensemble-graph-57d01-54cddf9746-zqjch namespace: kserve-ci-e2e-test resourceVersion: "20989" uid: c1fee85a-ce18-47e7-8f0a-74efdf5004d7 kind: Event lastTimestamp: "2026-07-02T14:38:35Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:e0bdb71677d5eb7ee4784378e591e8126d0dcafbbe991e5dcf178c67e85889cd" already present on machine metadata: creationTimestamp: "2026-07-02T14:38:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:35Z" name: ensemble-graph-57d01-54cddf9746-zqjch.18be7fe2f46c78af namespace: kserve-ci-e2e-test resourceVersion: "21013" uid: 9336376a-1204-4d21-add6-7924a8c5b283 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-57d01} kind: Pod name: ensemble-graph-57d01-54cddf9746-zqjch namespace: kserve-ci-e2e-test resourceVersion: "20989" uid: c1fee85a-ce18-47e7-8f0a-74efdf5004d7 kind: Event lastTimestamp: "2026-07-02T14:38:35Z" message: 'Created container: ensemble-graph-57d01' metadata: creationTimestamp: "2026-07-02T14:38:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:35Z" name: ensemble-graph-57d01-54cddf9746-zqjch.18be7fe2f7787003 namespace: kserve-ci-e2e-test resourceVersion: "21016" uid: c2c94e3e-0b62-42bc-a121-0c5cc8ffed36 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-57d01} kind: Pod name: ensemble-graph-57d01-54cddf9746-zqjch namespace: kserve-ci-e2e-test resourceVersion: "20989" uid: c1fee85a-ce18-47e7-8f0a-74efdf5004d7 kind: Event lastTimestamp: "2026-07-02T14:38:35Z" message: Started container ensemble-graph-57d01 metadata: creationTimestamp: "2026-07-02T14:38:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:35Z" name: ensemble-graph-57d01-54cddf9746-zqjch.18be7fe2f7ea8c81 namespace: kserve-ci-e2e-test resourceVersion: "21017" uid: d361ac1a-1f69-4c14-b628-62db47e20ba4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-57d01} kind: Pod name: ensemble-graph-57d01-54cddf9746-zqjch namespace: kserve-ci-e2e-test resourceVersion: "20989" uid: c1fee85a-ce18-47e7-8f0a-74efdf5004d7 kind: Event lastTimestamp: "2026-07-02T14:46:49Z" message: Stopping container ensemble-graph-57d01 metadata: creationTimestamp: "2026-07-02T14:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:46:49Z" name: ensemble-graph-57d01-54cddf9746-zqjch.18be805607cad761 namespace: kserve-ci-e2e-test resourceVersion: "24694" uid: c0b19fd1-b48b-466a-aee8-6dbf4604b273 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-57d01} kind: Pod name: ensemble-graph-57d01-54cddf9746-zqjch namespace: kserve-ci-e2e-test resourceVersion: "20989" uid: c1fee85a-ce18-47e7-8f0a-74efdf5004d7 kind: Event lastTimestamp: "2026-07-02T14:47:16Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:47:16Z" name: ensemble-graph-57d01-54cddf9746-zqjch.18be80568ae1ebf0 namespace: kserve-ci-e2e-test resourceVersion: "25034" uid: ebfdab10-f842-45cf-baf5-c44004b5b6ed reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-57d01-54cddf9746 namespace: kserve-ci-e2e-test resourceVersion: "20983" uid: 3a7dd9bb-4d03-4f7a-be8f-e23aad9177b4 kind: Event lastTimestamp: "2026-07-02T14:38:34Z" message: 'Created pod: ensemble-graph-57d01-54cddf9746-zqjch' metadata: creationTimestamp: "2026-07-02T14:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:34Z" name: ensemble-graph-57d01-54cddf9746.18be7fe2d8e2db1e namespace: kserve-ci-e2e-test resourceVersion: "20988" uid: de0b1300-67eb-4e2a-bdbd-86a3b9d82342 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-57d01 namespace: kserve-ci-e2e-test resourceVersion: "20980" uid: bf68b2ff-1fe9-4318-bfad-29b9642e0bb0 kind: Event lastTimestamp: "2026-07-02T14:38:34Z" message: Scaled up replica set ensemble-graph-57d01-54cddf9746 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:34Z" name: ensemble-graph-57d01.18be7fe2d82c7130 namespace: kserve-ci-e2e-test resourceVersion: "20985" uid: bd4642e6-df24-4ea8-b831-673ea94bae39 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:34Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-57d01 namespace: kserve-ci-e2e-test resourceVersion: "20998" uid: d492d7d2-0977-4cee-b60d-1b46442765b0 kind: Event lastTimestamp: "2026-07-02T14:38:34Z" message: InferenceGraph [ensemble-graph-57d01] is Ready metadata: creationTimestamp: "2026-07-02T14:38:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:38:34Z" name: ensemble-graph-57d01.18be7fe2dba2eba3 namespace: kserve-ci-e2e-test resourceVersion: "21000" uid: 445481a7-1bca-46d9-a64e-7edf4c7cdd9f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:38:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-57d01 namespace: kserve-ci-e2e-test resourceVersion: "20982" uid: 94b940e9-90f2-46e1-9958-3ea11bcfb5f7 kind: Event lastTimestamp: "2026-07-02T14:39:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:39:04Z" name: ensemble-graph-57d01.18be7fe65698619f namespace: kserve-ci-e2e-test resourceVersion: "21408" uid: 393938b1-0bc1-4002-a685-d07a8a6f7b4e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:38:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-57d01 namespace: kserve-ci-e2e-test resourceVersion: "20982" uid: 94b940e9-90f2-46e1-9958-3ea11bcfb5f7 kind: Event lastTimestamp: "2026-07-02T14:39:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:39:04Z" name: ensemble-graph-57d01.18be7fe656d567a1 namespace: kserve-ci-e2e-test resourceVersion: "21409" uid: c5a6061c-9d94-447c-8ad9-a6a28ca25781 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:37:24.468841Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-fbc8b-79dc459f7b-9p9rv namespace: kserve-ci-e2e-test resourceVersion: "20160" uid: db040183-bf2c-450e-9c88-7d6a63a87995 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-fbc8b-79dc459f7b-9p9rv to ip-10-0-143-249.ec2.internal metadata: creationTimestamp: "2026-07-02T14:37:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:37:24Z" name: ensemble-graph-fbc8b-79dc459f7b-9p9rv.18be7fd27b0176c2 namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: 80ecc6f6-0270-40bf-b38d-32849c7e42df reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:24Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-fbc8b-79dc459f7b-9p9rv namespace: kserve-ci-e2e-test resourceVersion: "20170" uid: db040183-bf2c-450e-9c88-7d6a63a87995 kind: Event lastTimestamp: "2026-07-02T14:37:24Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:37:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:37:24Z" name: ensemble-graph-fbc8b-79dc459f7b-9p9rv.18be7fd294771b69 namespace: kserve-ci-e2e-test resourceVersion: "20186" uid: 95e88aea-bab6-4518-afbd-540441f9b6de reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-fbc8b} kind: Pod name: ensemble-graph-fbc8b-79dc459f7b-9p9rv namespace: kserve-ci-e2e-test resourceVersion: "20163" uid: db040183-bf2c-450e-9c88-7d6a63a87995 kind: Event lastTimestamp: "2026-07-02T14:37:24Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:e0bdb71677d5eb7ee4784378e591e8126d0dcafbbe991e5dcf178c67e85889cd" already present on machine metadata: creationTimestamp: "2026-07-02T14:37:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:24Z" name: ensemble-graph-fbc8b-79dc459f7b-9p9rv.18be7fd295c35f43 namespace: kserve-ci-e2e-test resourceVersion: "20188" uid: 51283d61-a716-4d0d-8196-c52e4e3da3cf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-fbc8b} kind: Pod name: ensemble-graph-fbc8b-79dc459f7b-9p9rv namespace: kserve-ci-e2e-test resourceVersion: "20163" uid: db040183-bf2c-450e-9c88-7d6a63a87995 kind: Event lastTimestamp: "2026-07-02T14:37:24Z" message: 'Created container: ensemble-graph-fbc8b' metadata: creationTimestamp: "2026-07-02T14:37:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:24Z" name: ensemble-graph-fbc8b-79dc459f7b-9p9rv.18be7fd2987c4e35 namespace: kserve-ci-e2e-test resourceVersion: "20189" uid: 16f7a8fd-38e9-402c-8f0b-e6888971fca5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-fbc8b} kind: Pod name: ensemble-graph-fbc8b-79dc459f7b-9p9rv namespace: kserve-ci-e2e-test resourceVersion: "20163" uid: db040183-bf2c-450e-9c88-7d6a63a87995 kind: Event lastTimestamp: "2026-07-02T14:37:24Z" message: Started container ensemble-graph-fbc8b metadata: creationTimestamp: "2026-07-02T14:37:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:24Z" name: ensemble-graph-fbc8b-79dc459f7b-9p9rv.18be7fd298ec839b namespace: kserve-ci-e2e-test resourceVersion: "20190" uid: 75667aec-9f68-4b86-a573-b493e66852fc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-fbc8b} kind: Pod name: ensemble-graph-fbc8b-79dc459f7b-9p9rv namespace: kserve-ci-e2e-test resourceVersion: "20163" uid: db040183-bf2c-450e-9c88-7d6a63a87995 kind: Event lastTimestamp: "2026-07-02T14:37:34Z" message: Stopping container ensemble-graph-fbc8b metadata: creationTimestamp: "2026-07-02T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:34Z" name: ensemble-graph-fbc8b-79dc459f7b-9p9rv.18be7fd4d44c13ca namespace: kserve-ci-e2e-test resourceVersion: "20286" uid: 8ffdb554-0a0d-44c8-83e5-8acc34cae5b1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:37:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-fbc8b} kind: Pod name: ensemble-graph-fbc8b-79dc459f7b-9p9rv namespace: kserve-ci-e2e-test resourceVersion: "20163" uid: db040183-bf2c-450e-9c88-7d6a63a87995 kind: Event lastTimestamp: "2026-07-02T14:38:01Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:01Z" name: ensemble-graph-fbc8b-79dc459f7b-9p9rv.18be7fd5415d4627 namespace: kserve-ci-e2e-test resourceVersion: "20639" uid: 5ef5b8ed-bade-4c26-92d7-e15201134435 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-fbc8b-79dc459f7b namespace: kserve-ci-e2e-test resourceVersion: "20156" uid: 1e99619f-5dbe-49f7-945b-d6ec704da3b1 kind: Event lastTimestamp: "2026-07-02T14:37:24Z" message: 'Created pod: ensemble-graph-fbc8b-79dc459f7b-9p9rv' metadata: creationTimestamp: "2026-07-02T14:37:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:37:24Z" name: ensemble-graph-fbc8b-79dc459f7b.18be7fd27a8e9a48 namespace: kserve-ci-e2e-test resourceVersion: "20162" uid: 860f4986-f63f-4f8a-87bf-241d8cdf03f5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-fbc8b namespace: kserve-ci-e2e-test resourceVersion: "20154" uid: cc3b95db-a0d2-4302-8b30-95e4845c5f84 kind: Event lastTimestamp: "2026-07-02T14:37:24Z" message: Scaled up replica set ensemble-graph-fbc8b-79dc459f7b from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:37:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:37:24Z" name: ensemble-graph-fbc8b.18be7fd279d2c74a namespace: kserve-ci-e2e-test resourceVersion: "20158" uid: ef8c418a-0cac-46b6-beff-8a8d9fd0f941 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:24Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-fbc8b namespace: kserve-ci-e2e-test resourceVersion: "20172" uid: ca95b5c8-9d48-4db3-a40d-67faecdfbe4b kind: Event lastTimestamp: "2026-07-02T14:37:24Z" message: InferenceGraph [ensemble-graph-fbc8b] is Ready metadata: creationTimestamp: "2026-07-02T14:37:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:37:24Z" name: ensemble-graph-fbc8b.18be7fd27d90a763 namespace: kserve-ci-e2e-test resourceVersion: "20174" uid: 97d02aa9-5c63-4d2d-a2f6-2c80182820f8 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:27:09.554759Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1c2b7-predictor-6f6dfd7649-vf2fc namespace: kserve-ci-e2e-test resourceVersion: "15073" uid: 55158916-8aa3-4215-9c83-02b1224d1a62 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1c2b7-predictor-6f6dfd7649-vf2fc to ip-10-0-134-54.ec2.internal metadata: creationTimestamp: "2026-07-02T14:27:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:27:09Z" name: error-404-isvc-1c2b7-predictor-6f6dfd7649-vf2fc.18be7f434f453119 namespace: kserve-ci-e2e-test resourceVersion: "15079" uid: 1c9ae69f-5934-4da6-8cb0-c1d0398b750f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:09Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1c2b7-predictor-6f6dfd7649-vf2fc namespace: kserve-ci-e2e-test resourceVersion: "15090" uid: 55158916-8aa3-4215-9c83-02b1224d1a62 kind: Event lastTimestamp: "2026-07-02T14:27:09Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:27:09Z" name: error-404-isvc-1c2b7-predictor-6f6dfd7649-vf2fc.18be7f43694417f4 namespace: kserve-ci-e2e-test resourceVersion: "15114" uid: 9e514272-1364-41fc-97ba-d27f07bc36b3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1c2b7-predictor-6f6dfd7649-vf2fc namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 55158916-8aa3-4215-9c83-02b1224d1a62 kind: Event lastTimestamp: "2026-07-02T14:27:10Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:10Z" name: error-404-isvc-1c2b7-predictor-6f6dfd7649-vf2fc.18be7f436aeb4397 namespace: kserve-ci-e2e-test resourceVersion: "15119" uid: 84fcde6c-a36f-40f9-a97f-91cbb054061d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1c2b7-predictor-6f6dfd7649-vf2fc namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 55158916-8aa3-4215-9c83-02b1224d1a62 kind: Event lastTimestamp: "2026-07-02T14:27:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:10Z" name: error-404-isvc-1c2b7-predictor-6f6dfd7649-vf2fc.18be7f436e09a6e2 namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: 120485a4-02f3-4373-b28c-2c77577581aa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1c2b7-predictor-6f6dfd7649-vf2fc namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 55158916-8aa3-4215-9c83-02b1224d1a62 kind: Event lastTimestamp: "2026-07-02T14:27:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:10Z" name: error-404-isvc-1c2b7-predictor-6f6dfd7649-vf2fc.18be7f436e7a6094 namespace: kserve-ci-e2e-test resourceVersion: "15122" uid: 3dcc7ab1-2ef6-4953-a7ac-20583f0e608b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1c2b7-predictor-6f6dfd7649-vf2fc namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 55158916-8aa3-4215-9c83-02b1224d1a62 kind: Event lastTimestamp: "2026-07-02T14:27:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:10Z" name: error-404-isvc-1c2b7-predictor-6f6dfd7649-vf2fc.18be7f436e84129c namespace: kserve-ci-e2e-test resourceVersion: "15124" uid: 25e6ee12-de01-4297-a9c2-6cdba7f7c383 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1c2b7-predictor-6f6dfd7649-vf2fc namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 55158916-8aa3-4215-9c83-02b1224d1a62 kind: Event lastTimestamp: "2026-07-02T14:27:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:10Z" name: error-404-isvc-1c2b7-predictor-6f6dfd7649-vf2fc.18be7f43715a6db2 namespace: kserve-ci-e2e-test resourceVersion: "15125" uid: 1fc4f02a-d1a0-42ea-a84d-f7dadcb619e7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1c2b7-predictor-6f6dfd7649-vf2fc namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 55158916-8aa3-4215-9c83-02b1224d1a62 kind: Event lastTimestamp: "2026-07-02T14:27:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:10Z" name: error-404-isvc-1c2b7-predictor-6f6dfd7649-vf2fc.18be7f4371cc5584 namespace: kserve-ci-e2e-test resourceVersion: "15126" uid: 4967ca24-7d7d-4c84-8837-fadb5c2a6f8c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T14:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1c2b7-predictor-6f6dfd7649-vf2fc namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 55158916-8aa3-4215-9c83-02b1224d1a62 kind: Event lastTimestamp: "2026-07-02T14:36:26Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:26Z" name: error-404-isvc-1c2b7-predictor-6f6dfd7649-vf2fc.18be7f4377165b17 namespace: kserve-ci-e2e-test resourceVersion: "19543" uid: 91f14433-69cc-4b8b-96c6-2ff14f45832f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1c2b7-predictor-6f6dfd7649-vf2fc namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 55158916-8aa3-4215-9c83-02b1224d1a62 kind: Event lastTimestamp: "2026-07-02T14:36:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:24Z" name: error-404-isvc-1c2b7-predictor-6f6dfd7649-vf2fc.18be7fc48ba4439d namespace: kserve-ci-e2e-test resourceVersion: "19496" uid: 9ebf09b2-7c4e-4e03-9de2-5f4985c37b59 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1c2b7-predictor-6f6dfd7649-vf2fc namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 55158916-8aa3-4215-9c83-02b1224d1a62 kind: Event lastTimestamp: "2026-07-02T14:36:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:24Z" name: error-404-isvc-1c2b7-predictor-6f6dfd7649-vf2fc.18be7fc48ba4c35e namespace: kserve-ci-e2e-test resourceVersion: "19497" uid: a7816aa7-1c6c-4906-b579-a9f32e6b5097 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1c2b7-predictor-6f6dfd7649-vf2fc namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 55158916-8aa3-4215-9c83-02b1224d1a62 kind: Event lastTimestamp: "2026-07-02T14:36:26Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:26Z" name: error-404-isvc-1c2b7-predictor-6f6dfd7649-vf2fc.18be7fc4ebb48511 namespace: kserve-ci-e2e-test resourceVersion: "19542" uid: 62e9b988-649d-437a-a4c2-e9460152776c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1c2b7-predictor-6f6dfd7649 namespace: kserve-ci-e2e-test resourceVersion: "15064" uid: 9553ac51-ef55-4390-9876-d9963d0eb289 kind: Event lastTimestamp: "2026-07-02T14:27:09Z" message: 'Created pod: error-404-isvc-1c2b7-predictor-6f6dfd7649-vf2fc' metadata: creationTimestamp: "2026-07-02T14:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:09Z" name: error-404-isvc-1c2b7-predictor-6f6dfd7649.18be7f434eb374f0 namespace: kserve-ci-e2e-test resourceVersion: "15074" uid: f51de163-1a34-4d2b-a076-23d7758590f7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1c2b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "15061" uid: ec064c55-75b4-415b-834b-d013e147fb70 kind: Event lastTimestamp: "2026-07-02T14:27:09Z" message: Scaled up replica set error-404-isvc-1c2b7-predictor-6f6dfd7649 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:09Z" name: error-404-isvc-1c2b7-predictor.18be7f434d14f42a namespace: kserve-ci-e2e-test resourceVersion: "15067" uid: a68284f0-22a5-4e50-8ce6-164d397d42f2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:27:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1c2b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "15063" uid: 82c5485d-aaea-4385-b891-7e732aba5d61 kind: Event lastTimestamp: "2026-07-02T14:27:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:27:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:54Z" name: error-404-isvc-1c2b7-predictor.18be7f46cb8c0084 namespace: kserve-ci-e2e-test resourceVersion: "15522" uid: 59feca72-180f-4f8d-89ab-2911b08efa3b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:27:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1c2b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "15063" uid: 82c5485d-aaea-4385-b891-7e732aba5d61 kind: Event lastTimestamp: "2026-07-02T14:27:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:27:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:54Z" name: error-404-isvc-1c2b7-predictor.18be7f46cbeaefbb namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: 084e7b2e-af2a-468e-86b4-4da63d0040dd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:28:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1c2b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "15271" uid: 82c5485d-aaea-4385-b891-7e732aba5d61 kind: Event lastTimestamp: "2026-07-02T14:28:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:24Z" name: error-404-isvc-1c2b7-predictor.18be7f514863d8f2 namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: 2388418e-d1e5-4d97-b117-f3f09dd5893d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:28:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1c2b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "15271" uid: 82c5485d-aaea-4385-b891-7e732aba5d61 kind: Event lastTimestamp: "2026-07-02T14:28:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:24Z" name: error-404-isvc-1c2b7-predictor.18be7f51490eb550 namespace: kserve-ci-e2e-test resourceVersion: "16036" uid: 7d303c9f-6997-4047-ab54-1e09d5b510d7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:27:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1c2b7 namespace: kserve-ci-e2e-test resourceVersion: "15031" uid: a5abb226-a053-4c64-b746-a737efddcf31 kind: Event lastTimestamp: "2026-07-02T14:27:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:27:56Z" name: error-404-isvc-1c2b7.18be7f434d44d915 namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: 9c838643-f7a0-4307-a708-2f9fcb998fd1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1c2b7 namespace: kserve-ci-e2e-test resourceVersion: "15031" uid: a5abb226-a053-4c64-b746-a737efddcf31 kind: Event lastTimestamp: "2026-07-02T14:27:09Z" message: 'Failed to update status for InferenceService "error-404-isvc-1c2b7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1c2b7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:27:09Z" name: error-404-isvc-1c2b7.18be7f434daa6448 namespace: kserve-ci-e2e-test resourceVersion: "15070" uid: e2abee78-a05c-4ca4-bdf9-4e7aeacd5272 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1c2b7 namespace: kserve-ci-e2e-test resourceVersion: "15031" uid: a5abb226-a053-4c64-b746-a737efddcf31 kind: Event lastTimestamp: "2026-07-02T14:27:09Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1c2b7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:27:09Z" name: error-404-isvc-1c2b7.18be7f434daac2ef namespace: kserve-ci-e2e-test resourceVersion: "15072" uid: 9da2411b-18a1-4b22-8886-55f29626ba6e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1c2b7 namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: a5abb226-a053-4c64-b746-a737efddcf31 kind: Event lastTimestamp: "2026-07-02T14:27:56Z" message: InferenceService [error-404-isvc-1c2b7] is Ready metadata: creationTimestamp: "2026-07-02T14:27:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:27:56Z" name: error-404-isvc-1c2b7.18be7f4e334850d5 namespace: kserve-ci-e2e-test resourceVersion: "15585" uid: 25628845-7936-4072-bae3-33ba490d2a34 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:48:00.058280Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2963e-predictor-6bb9dd4685-ds5fb namespace: kserve-ci-e2e-test resourceVersion: "25610" uid: c220f358-6343-4525-880a-e0ebfb69e173 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-2963e-predictor-6bb9dd4685-ds5fb to ip-10-0-134-54.ec2.internal metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:48:00Z" name: error-404-isvc-2963e-predictor-6bb9dd4685-ds5fb.18be80667716ac70 namespace: kserve-ci-e2e-test resourceVersion: "25616" uid: 21dcb08f-de7c-42ae-b937-a45b12bd7bb5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2963e-predictor-6bb9dd4685-ds5fb namespace: kserve-ci-e2e-test resourceVersion: "25621" uid: c220f358-6343-4525-880a-e0ebfb69e173 kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:48:00Z" name: error-404-isvc-2963e-predictor-6bb9dd4685-ds5fb.18be8066981e6426 namespace: kserve-ci-e2e-test resourceVersion: "25653" uid: db2a7832-ae30-48c4-a671-3c9bb59da01d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2963e-predictor-6bb9dd4685-ds5fb namespace: kserve-ci-e2e-test resourceVersion: "25613" uid: c220f358-6343-4525-880a-e0ebfb69e173 kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:00Z" name: error-404-isvc-2963e-predictor-6bb9dd4685-ds5fb.18be806699c325cf namespace: kserve-ci-e2e-test resourceVersion: "25655" uid: d8560743-4194-4feb-bfef-0a32aceea7c7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2963e-predictor-6bb9dd4685-ds5fb namespace: kserve-ci-e2e-test resourceVersion: "25613" uid: c220f358-6343-4525-880a-e0ebfb69e173 kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:00Z" name: error-404-isvc-2963e-predictor-6bb9dd4685-ds5fb.18be80669c0bf1ca namespace: kserve-ci-e2e-test resourceVersion: "25658" uid: 72577fd3-6b24-4fd6-b2e6-955f9d53b8e1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2963e-predictor-6bb9dd4685-ds5fb namespace: kserve-ci-e2e-test resourceVersion: "25613" uid: c220f358-6343-4525-880a-e0ebfb69e173 kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:00Z" name: error-404-isvc-2963e-predictor-6bb9dd4685-ds5fb.18be80669d0c6c92 namespace: kserve-ci-e2e-test resourceVersion: "25660" uid: 8e7b2616-b223-4a51-a7b5-9641a0bf1a23 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2963e-predictor-6bb9dd4685-ds5fb namespace: kserve-ci-e2e-test resourceVersion: "25613" uid: c220f358-6343-4525-880a-e0ebfb69e173 kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:00Z" name: error-404-isvc-2963e-predictor-6bb9dd4685-ds5fb.18be80669d1bff7c namespace: kserve-ci-e2e-test resourceVersion: "25661" uid: e9c5237f-5376-41d1-a007-16d3bd70c40b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2963e-predictor-6bb9dd4685-ds5fb namespace: kserve-ci-e2e-test resourceVersion: "25613" uid: c220f358-6343-4525-880a-e0ebfb69e173 kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:00Z" name: error-404-isvc-2963e-predictor-6bb9dd4685-ds5fb.18be80669fe1490a namespace: kserve-ci-e2e-test resourceVersion: "25663" uid: f18497bd-65b0-43f8-9802-322e16c4d10e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2963e-predictor-6bb9dd4685-ds5fb namespace: kserve-ci-e2e-test resourceVersion: "25613" uid: c220f358-6343-4525-880a-e0ebfb69e173 kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:00Z" name: error-404-isvc-2963e-predictor-6bb9dd4685-ds5fb.18be8066a057aa44 namespace: kserve-ci-e2e-test resourceVersion: "25664" uid: a2db44cf-59d5-4f6f-ab85-6e7a2650bc09 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-02T14:48:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2963e-predictor-6bb9dd4685-ds5fb namespace: kserve-ci-e2e-test resourceVersion: "25613" uid: c220f358-6343-4525-880a-e0ebfb69e173 kind: Event lastTimestamp: "2026-07-02T14:57:27Z" message: 'Readiness probe failed: dial tcp 10.133.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:57:27Z" name: error-404-isvc-2963e-predictor-6bb9dd4685-ds5fb.18be8066b9472c7c namespace: kserve-ci-e2e-test resourceVersion: "30184" uid: a02f0f8f-7a15-4889-95ec-f378996e38c0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:57:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2963e-predictor-6bb9dd4685-ds5fb namespace: kserve-ci-e2e-test resourceVersion: "25613" uid: c220f358-6343-4525-880a-e0ebfb69e173 kind: Event lastTimestamp: "2026-07-02T14:57:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:57:24Z" name: error-404-isvc-2963e-predictor-6bb9dd4685-ds5fb.18be80e9f328301e namespace: kserve-ci-e2e-test resourceVersion: "30162" uid: e5ee8e07-ad36-405a-8610-6970b13dd2ad reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:57:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2963e-predictor-6bb9dd4685-ds5fb namespace: kserve-ci-e2e-test resourceVersion: "25613" uid: c220f358-6343-4525-880a-e0ebfb69e173 kind: Event lastTimestamp: "2026-07-02T14:57:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:57:24Z" name: error-404-isvc-2963e-predictor-6bb9dd4685-ds5fb.18be80e9f328bf3e namespace: kserve-ci-e2e-test resourceVersion: "30163" uid: 177773f2-9e28-4ef0-84cc-4cbacfdf9fe6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:57:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2963e-predictor-6bb9dd4685-ds5fb namespace: kserve-ci-e2e-test resourceVersion: "25613" uid: c220f358-6343-4525-880a-e0ebfb69e173 kind: Event lastTimestamp: "2026-07-02T14:57:27Z" message: 'Readiness probe failed: Get "https://10.133.0.41:8643/healthz": dial tcp 10.133.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:57:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:57:27Z" name: error-404-isvc-2963e-predictor-6bb9dd4685-ds5fb.18be80ea82088b5e namespace: kserve-ci-e2e-test resourceVersion: "30181" uid: 43df69fd-d814-4ebc-818c-33a785fcf959 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-2963e-predictor-6bb9dd4685 namespace: kserve-ci-e2e-test resourceVersion: "25602" uid: afce4fb0-4559-4144-8273-4c2116b2950c kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: 'Created pod: error-404-isvc-2963e-predictor-6bb9dd4685-ds5fb' metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:48:00Z" name: error-404-isvc-2963e-predictor-6bb9dd4685.18be8066769a9346 namespace: kserve-ci-e2e-test resourceVersion: "25612" uid: a76f27fe-e36b-4def-92e5-f65f09884702 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-2963e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25598" uid: f3c9678f-54ef-4b43-ae9a-dec202f5eb65 kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: Scaled up replica set error-404-isvc-2963e-predictor-6bb9dd4685 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:48:00Z" name: error-404-isvc-2963e-predictor.18be806674ca9cec namespace: kserve-ci-e2e-test resourceVersion: "25604" uid: 84eb29eb-7211-4504-a1e7-905d235b74f5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2963e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25601" uid: 36a2ab25-0010-42ab-916f-6f16e8ba0087 kind: Event lastTimestamp: "2026-07-02T14:48:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:48:15Z" name: error-404-isvc-2963e-predictor.18be8069f35b6b83 namespace: kserve-ci-e2e-test resourceVersion: "25829" uid: 5c069865-7fa7-4714-925c-0f426483fbae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2963e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25601" uid: 36a2ab25-0010-42ab-916f-6f16e8ba0087 kind: Event lastTimestamp: "2026-07-02T14:48:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:48:15Z" name: error-404-isvc-2963e-predictor.18be8069f3cf32e0 namespace: kserve-ci-e2e-test resourceVersion: "25831" uid: d3b2a4e4-d05b-44f4-ba31-41656dbc24e7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:48:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2963e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25830" uid: 36a2ab25-0010-42ab-916f-6f16e8ba0087 kind: Event lastTimestamp: "2026-07-02T14:49:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:49:30Z" name: error-404-isvc-2963e-predictor.18be806d729d4624 namespace: kserve-ci-e2e-test resourceVersion: "26831" uid: 94687c20-8a2c-48cb-a412-a42765ca386a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:48:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2963e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25830" uid: 36a2ab25-0010-42ab-916f-6f16e8ba0087 kind: Event lastTimestamp: "2026-07-02T14:49:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:49:30Z" name: error-404-isvc-2963e-predictor.18be806d7316c746 namespace: kserve-ci-e2e-test resourceVersion: "26832" uid: 7cd1bd7f-868a-411a-8bd1-a6f0ad69d147 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2963e namespace: kserve-ci-e2e-test resourceVersion: "25585" uid: 03f027d6-f81f-4645-b072-0777b59ccdef kind: Event lastTimestamp: "2026-07-02T14:48:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:48:57Z" name: error-404-isvc-2963e.18be806674e13c85 namespace: kserve-ci-e2e-test resourceVersion: "26493" uid: b9bb5f86-4d08-4d82-8d23-720c9fa0dd6b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2963e namespace: kserve-ci-e2e-test resourceVersion: "25585" uid: 03f027d6-f81f-4645-b072-0777b59ccdef kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: 'Failed to update status for InferenceService "error-404-isvc-2963e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2963e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:48:00Z" name: error-404-isvc-2963e.18be8066756df377 namespace: kserve-ci-e2e-test resourceVersion: "25607" uid: 9e389bdb-4608-491c-bab7-289f45e18899 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2963e namespace: kserve-ci-e2e-test resourceVersion: "25585" uid: 03f027d6-f81f-4645-b072-0777b59ccdef kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2963e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:48:00Z" name: error-404-isvc-2963e.18be8066756e6fc1 namespace: kserve-ci-e2e-test resourceVersion: "25609" uid: f1617b71-e415-443e-9555-c1c8b17d75ff reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2963e namespace: kserve-ci-e2e-test resourceVersion: "26485" uid: 03f027d6-f81f-4645-b072-0777b59ccdef kind: Event lastTimestamp: "2026-07-02T14:48:57Z" message: InferenceService [error-404-isvc-2963e] is Ready metadata: creationTimestamp: "2026-07-02T14:48:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:48:57Z" name: error-404-isvc-2963e.18be8073d1278b9c namespace: kserve-ci-e2e-test resourceVersion: "26486" uid: 4601cf35-126f-40ff-aaf5-09eccfb68823 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:46:49.736975Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9 namespace: kserve-ci-e2e-test resourceVersion: "24758" uid: ce9256f2-780e-40cc-bfb8-343cbdf42f3f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-484ca-predictor-9b4fd8447-xvlc9 to ip-10-0-134-54.ec2.internal metadata: creationTimestamp: "2026-07-02T14:46:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:46:49Z" name: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9.18be8056179ce501 namespace: kserve-ci-e2e-test resourceVersion: "24763" uid: 086d9de4-4ed0-4e79-85a0-9d5dc9b4e530 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:49Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9 namespace: kserve-ci-e2e-test resourceVersion: "24760" uid: ce9256f2-780e-40cc-bfb8-343cbdf42f3f kind: Event lastTimestamp: "2026-07-02T14:46:49Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-484ca-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:46:49Z" name: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9.18be805626ecc87f namespace: kserve-ci-e2e-test resourceVersion: "24793" uid: e6b50e16-12fd-4022-831d-250749461188 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:50Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9 namespace: kserve-ci-e2e-test resourceVersion: "24773" uid: ce9256f2-780e-40cc-bfb8-343cbdf42f3f kind: Event lastTimestamp: "2026-07-02T14:46:50Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:46:50Z" name: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9.18be805655cab49b namespace: kserve-ci-e2e-test resourceVersion: "24809" uid: 2e001a73-2642-4499-81f1-c816fce74e62 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9 namespace: kserve-ci-e2e-test resourceVersion: "24760" uid: ce9256f2-780e-40cc-bfb8-343cbdf42f3f kind: Event lastTimestamp: "2026-07-02T14:46:50Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:46:50Z" name: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9.18be80565768b62e namespace: kserve-ci-e2e-test resourceVersion: "24814" uid: 96e43471-f614-484f-ab42-9dd17c27d0dd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9 namespace: kserve-ci-e2e-test resourceVersion: "24760" uid: ce9256f2-780e-40cc-bfb8-343cbdf42f3f kind: Event lastTimestamp: "2026-07-02T14:46:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:46:50Z" name: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9.18be80565a18b72e namespace: kserve-ci-e2e-test resourceVersion: "24816" uid: 0cb8f707-9344-4b21-bbcd-1c18ca00de0b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9 namespace: kserve-ci-e2e-test resourceVersion: "24760" uid: ce9256f2-780e-40cc-bfb8-343cbdf42f3f kind: Event lastTimestamp: "2026-07-02T14:46:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:46:50Z" name: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9.18be80565a887f75 namespace: kserve-ci-e2e-test resourceVersion: "24817" uid: 34f74e86-f1e1-43a7-b854-7b68433dc49a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9 namespace: kserve-ci-e2e-test resourceVersion: "24760" uid: ce9256f2-780e-40cc-bfb8-343cbdf42f3f kind: Event lastTimestamp: "2026-07-02T14:46:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:46:50Z" name: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9.18be80565a909392 namespace: kserve-ci-e2e-test resourceVersion: "24818" uid: d91a377b-291c-4293-a20f-709b01a8c0d8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9 namespace: kserve-ci-e2e-test resourceVersion: "24760" uid: ce9256f2-780e-40cc-bfb8-343cbdf42f3f kind: Event lastTimestamp: "2026-07-02T14:46:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:46:50Z" name: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9.18be80565e3a8be3 namespace: kserve-ci-e2e-test resourceVersion: "24821" uid: 11d9ff58-3027-425c-b7a7-b2eabb0e45dc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9 namespace: kserve-ci-e2e-test resourceVersion: "24760" uid: ce9256f2-780e-40cc-bfb8-343cbdf42f3f kind: Event lastTimestamp: "2026-07-02T14:46:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:46:50Z" name: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9.18be80565eaaa871 namespace: kserve-ci-e2e-test resourceVersion: "24822" uid: b1fd3f95-99de-4879-a358-d757c3372f38 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9 namespace: kserve-ci-e2e-test resourceVersion: "24760" uid: ce9256f2-780e-40cc-bfb8-343cbdf42f3f kind: Event lastTimestamp: "2026-07-02T14:47:27Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:47:27Z" name: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9.18be805696fb0dcd namespace: kserve-ci-e2e-test resourceVersion: "25119" uid: f027decf-7589-4466-b708-6fe9ca4a00bf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9 namespace: kserve-ci-e2e-test resourceVersion: "24760" uid: ce9256f2-780e-40cc-bfb8-343cbdf42f3f kind: Event lastTimestamp: "2026-07-02T14:47:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:47:59Z" name: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9.18be80666c32c162 namespace: kserve-ci-e2e-test resourceVersion: "25552" uid: 489fbd8e-a9f7-430e-a7a9-ebc68dddb5c2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9 namespace: kserve-ci-e2e-test resourceVersion: "24760" uid: ce9256f2-780e-40cc-bfb8-343cbdf42f3f kind: Event lastTimestamp: "2026-07-02T14:47:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:47:59Z" name: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9.18be80666c34b020 namespace: kserve-ci-e2e-test resourceVersion: "25555" uid: 426aca52-e488-4afc-9bb3-0a65a1624197 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9 namespace: kserve-ci-e2e-test resourceVersion: "24760" uid: ce9256f2-780e-40cc-bfb8-343cbdf42f3f kind: Event lastTimestamp: "2026-07-02T14:48:02Z" message: 'Readiness probe failed: Get "https://10.133.0.37:8643/healthz": dial tcp 10.133.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:48:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:02Z" name: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9.18be80671f26dfca namespace: kserve-ci-e2e-test resourceVersion: "25704" uid: 292099b1-a108-4952-935f-9ec0484cf7f6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-484ca-predictor-9b4fd8447 namespace: kserve-ci-e2e-test resourceVersion: "24751" uid: 10ec9a69-68aa-4bbf-9a0a-d67aa5ba9e0c kind: Event lastTimestamp: "2026-07-02T14:46:49Z" message: 'Created pod: error-404-isvc-484ca-predictor-9b4fd8447-xvlc9' metadata: creationTimestamp: "2026-07-02T14:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:46:49Z" name: error-404-isvc-484ca-predictor-9b4fd8447.18be80561725f139 namespace: kserve-ci-e2e-test resourceVersion: "24759" uid: 43ab7ad4-2739-451a-88cd-1829536bc941 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-484ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "24747" uid: 558f0652-c60f-42d6-976a-d622c0ef529d kind: Event lastTimestamp: "2026-07-02T14:46:49Z" message: Scaled up replica set error-404-isvc-484ca-predictor-9b4fd8447 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:46:49Z" name: error-404-isvc-484ca-predictor.18be80561571bf21 namespace: kserve-ci-e2e-test resourceVersion: "24752" uid: 99162a1c-bd1b-4240-bec6-6c6907c90aba reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:47:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-484ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "24750" uid: 1dbc47be-a379-4e11-9d90-6d1134e6a853 kind: Event lastTimestamp: "2026-07-02T14:47:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:47:19Z" name: error-404-isvc-484ca-predictor.18be8059940932f6 namespace: kserve-ci-e2e-test resourceVersion: "25057" uid: 1f4305e8-9b55-45e7-8a5c-f4168489b683 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:47:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-484ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "24750" uid: 1dbc47be-a379-4e11-9d90-6d1134e6a853 kind: Event lastTimestamp: "2026-07-02T14:47:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:47:19Z" name: error-404-isvc-484ca-predictor.18be80599454a997 namespace: kserve-ci-e2e-test resourceVersion: "25058" uid: 5534f6f5-cdd1-4964-a933-00f98ebbf531 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:47:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-484ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "24946" uid: 1dbc47be-a379-4e11-9d90-6d1134e6a853 kind: Event lastTimestamp: "2026-07-02T14:47:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:47:49Z" name: error-404-isvc-484ca-predictor.18be806091b47c99 namespace: kserve-ci-e2e-test resourceVersion: "25310" uid: 1353e781-c6a9-4948-adc7-0a6036e31bda reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:47:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-484ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "24946" uid: 1dbc47be-a379-4e11-9d90-6d1134e6a853 kind: Event lastTimestamp: "2026-07-02T14:47:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:47:49Z" name: error-404-isvc-484ca-predictor.18be80609221c8ef namespace: kserve-ci-e2e-test resourceVersion: "25311" uid: a393a41e-ad66-4c3d-96c3-6b904fc01dca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:46:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-484ca namespace: kserve-ci-e2e-test resourceVersion: "24732" uid: 3de2b40d-e2dc-41fd-b4d3-49c86bbaa33b kind: Event lastTimestamp: "2026-07-02T14:47:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:47:38Z" name: error-404-isvc-484ca.18be805615d889cd namespace: kserve-ci-e2e-test resourceVersion: "25207" uid: a9189ba5-9358-4cd9-b3f3-5ab9d95b457a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-484ca namespace: kserve-ci-e2e-test resourceVersion: "24732" uid: 3de2b40d-e2dc-41fd-b4d3-49c86bbaa33b kind: Event lastTimestamp: "2026-07-02T14:46:49Z" message: 'Failed to update status for InferenceService "error-404-isvc-484ca": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-484ca": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:46:49Z" name: error-404-isvc-484ca.18be80561631c730 namespace: kserve-ci-e2e-test resourceVersion: "24755" uid: 11fc5e9e-2bb2-43bd-9b71-c8e252d23ab1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-484ca namespace: kserve-ci-e2e-test resourceVersion: "24732" uid: 3de2b40d-e2dc-41fd-b4d3-49c86bbaa33b kind: Event lastTimestamp: "2026-07-02T14:46:49Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-484ca": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:46:49Z" name: error-404-isvc-484ca.18be805616322ba8 namespace: kserve-ci-e2e-test resourceVersion: "24757" uid: 6aec56bd-2489-4600-be88-af5e99510102 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-484ca namespace: kserve-ci-e2e-test resourceVersion: "25202" uid: 3de2b40d-e2dc-41fd-b4d3-49c86bbaa33b kind: Event lastTimestamp: "2026-07-02T14:47:37Z" message: InferenceService [error-404-isvc-484ca] is Ready metadata: creationTimestamp: "2026-07-02T14:47:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:47:37Z" name: error-404-isvc-484ca.18be806152c58fcd namespace: kserve-ci-e2e-test resourceVersion: "25203" uid: b9ce4d24-96a8-40a4-93de-56a91647ff04 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:37:34.816199Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-57d01-predictor-7b7d9b5777-88mnx namespace: kserve-ci-e2e-test resourceVersion: "20352" uid: 8c2fe9bf-8edd-416f-b699-67c932039ec6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-57d01-predictor-7b7d9b5777-88mnx to ip-10-0-134-54.ec2.internal metadata: creationTimestamp: "2026-07-02T14:37:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:37:34Z" name: error-404-isvc-57d01-predictor-7b7d9b5777-88mnx.18be7fd4e3c18d31 namespace: kserve-ci-e2e-test resourceVersion: "20358" uid: ea661832-0db5-4bb1-b4a4-6cc3976585aa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:35Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-57d01-predictor-7b7d9b5777-88mnx namespace: kserve-ci-e2e-test resourceVersion: "20368" uid: 8c2fe9bf-8edd-416f-b699-67c932039ec6 kind: Event lastTimestamp: "2026-07-02T14:37:35Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:37:35Z" name: error-404-isvc-57d01-predictor-7b7d9b5777-88mnx.18be7fd4fd665623 namespace: kserve-ci-e2e-test resourceVersion: "20393" uid: 0a4e60ac-9ab1-4eee-8bde-e9ff3546881b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-57d01-predictor-7b7d9b5777-88mnx namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 8c2fe9bf-8edd-416f-b699-67c932039ec6 kind: Event lastTimestamp: "2026-07-02T14:37:35Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:35Z" name: error-404-isvc-57d01-predictor-7b7d9b5777-88mnx.18be7fd4feb5e59f namespace: kserve-ci-e2e-test resourceVersion: "20395" uid: 481e830f-1696-4607-83fc-a6605d391f26 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-57d01-predictor-7b7d9b5777-88mnx namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 8c2fe9bf-8edd-416f-b699-67c932039ec6 kind: Event lastTimestamp: "2026-07-02T14:37:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:35Z" name: error-404-isvc-57d01-predictor-7b7d9b5777-88mnx.18be7fd5024d0e30 namespace: kserve-ci-e2e-test resourceVersion: "20400" uid: 37967f5c-4e5c-4f4f-9efa-f05002992496 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-57d01-predictor-7b7d9b5777-88mnx namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 8c2fe9bf-8edd-416f-b699-67c932039ec6 kind: Event lastTimestamp: "2026-07-02T14:37:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:35Z" name: error-404-isvc-57d01-predictor-7b7d9b5777-88mnx.18be7fd502bdd46d namespace: kserve-ci-e2e-test resourceVersion: "20401" uid: a4190096-446e-4294-9b71-6f32838a7d0a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-57d01-predictor-7b7d9b5777-88mnx namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 8c2fe9bf-8edd-416f-b699-67c932039ec6 kind: Event lastTimestamp: "2026-07-02T14:37:35Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:35Z" name: error-404-isvc-57d01-predictor-7b7d9b5777-88mnx.18be7fd502c77837 namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: c31e2d29-6502-4f63-a5c2-a3e1e1bc714a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-57d01-predictor-7b7d9b5777-88mnx namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 8c2fe9bf-8edd-416f-b699-67c932039ec6 kind: Event lastTimestamp: "2026-07-02T14:37:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:35Z" name: error-404-isvc-57d01-predictor-7b7d9b5777-88mnx.18be7fd506557f44 namespace: kserve-ci-e2e-test resourceVersion: "20403" uid: b83a5e9c-c72d-4373-ae8a-5d4fdc9ce946 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-57d01-predictor-7b7d9b5777-88mnx namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 8c2fe9bf-8edd-416f-b699-67c932039ec6 kind: Event lastTimestamp: "2026-07-02T14:37:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:35Z" name: error-404-isvc-57d01-predictor-7b7d9b5777-88mnx.18be7fd506cb8c3a namespace: kserve-ci-e2e-test resourceVersion: "20404" uid: 21695dda-9070-47ef-971c-46b8ca8790fa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-02T14:37:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-57d01-predictor-7b7d9b5777-88mnx namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 8c2fe9bf-8edd-416f-b699-67c932039ec6 kind: Event lastTimestamp: "2026-07-02T14:46:52Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:46:52Z" name: error-404-isvc-57d01-predictor-7b7d9b5777-88mnx.18be7fd554fa9297 namespace: kserve-ci-e2e-test resourceVersion: "24845" uid: e62bc0d4-4e32-406a-a942-4dfebe9cb6a3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-57d01-predictor-7b7d9b5777-88mnx namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 8c2fe9bf-8edd-416f-b699-67c932039ec6 kind: Event lastTimestamp: "2026-07-02T14:46:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:46:49Z" name: error-404-isvc-57d01-predictor-7b7d9b5777-88mnx.18be80561a630906 namespace: kserve-ci-e2e-test resourceVersion: "24781" uid: 3e7bdb06-dc05-49a7-a7ec-814966196fb3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-57d01-predictor-7b7d9b5777-88mnx namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 8c2fe9bf-8edd-416f-b699-67c932039ec6 kind: Event lastTimestamp: "2026-07-02T14:46:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:46:49Z" name: error-404-isvc-57d01-predictor-7b7d9b5777-88mnx.18be80561a652cbb namespace: kserve-ci-e2e-test resourceVersion: "24782" uid: 82e0b9d4-d5af-4b27-92b1-b941c36249f9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-57d01-predictor-7b7d9b5777-88mnx namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 8c2fe9bf-8edd-416f-b699-67c932039ec6 kind: Event lastTimestamp: "2026-07-02T14:46:52Z" message: 'Readiness probe failed: Get "https://10.133.0.33:8643/healthz": dial tcp 10.133.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:46:52Z" name: error-404-isvc-57d01-predictor-7b7d9b5777-88mnx.18be8056c95eea0d namespace: kserve-ci-e2e-test resourceVersion: "24842" uid: 9124ca28-8386-483d-857a-1c0530cc24dd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-57d01-predictor-7b7d9b5777 namespace: kserve-ci-e2e-test resourceVersion: "20344" uid: 7814460e-6e22-44e5-8a86-a3a93e2ddd6b kind: Event lastTimestamp: "2026-07-02T14:37:34Z" message: 'Created pod: error-404-isvc-57d01-predictor-7b7d9b5777-88mnx' metadata: creationTimestamp: "2026-07-02T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:37:34Z" name: error-404-isvc-57d01-predictor-7b7d9b5777.18be7fd4e33ccb1b namespace: kserve-ci-e2e-test resourceVersion: "20354" uid: b6cb09d4-b624-444f-abda-037aae08b11f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-57d01-predictor namespace: kserve-ci-e2e-test resourceVersion: "20342" uid: ffe38193-4ddf-4093-8bba-f4dcae2f6360 kind: Event lastTimestamp: "2026-07-02T14:37:34Z" message: Scaled up replica set error-404-isvc-57d01-predictor-7b7d9b5777 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:37:34Z" name: error-404-isvc-57d01-predictor.18be7fd4e199be4a namespace: kserve-ci-e2e-test resourceVersion: "20345" uid: 46a7b6d0-7140-4811-b330-06474c866e9c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-57d01-predictor namespace: kserve-ci-e2e-test resourceVersion: "20343" uid: eb51890d-2a98-47d1-acbf-57ef2cee8bff kind: Event lastTimestamp: "2026-07-02T14:37:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:37:49Z" name: error-404-isvc-57d01-predictor.18be7fd860101b0c namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: 7a0ea279-b85e-4e16-8def-e509a208a68a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-57d01-predictor namespace: kserve-ci-e2e-test resourceVersion: "20343" uid: eb51890d-2a98-47d1-acbf-57ef2cee8bff kind: Event lastTimestamp: "2026-07-02T14:37:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:37:49Z" name: error-404-isvc-57d01-predictor.18be7fd8607a3a94 namespace: kserve-ci-e2e-test resourceVersion: "20548" uid: f2583d9f-1be9-4f69-8c2c-21ff4e09e155 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:38:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-57d01-predictor namespace: kserve-ci-e2e-test resourceVersion: "20547" uid: eb51890d-2a98-47d1-acbf-57ef2cee8bff kind: Event lastTimestamp: "2026-07-02T14:39:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:39:04Z" name: error-404-isvc-57d01-predictor.18be7fdbdf43ca1f namespace: kserve-ci-e2e-test resourceVersion: "21412" uid: 7cf15b23-981f-4eca-9880-1eecae55f89f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:38:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-57d01-predictor namespace: kserve-ci-e2e-test resourceVersion: "20547" uid: eb51890d-2a98-47d1-acbf-57ef2cee8bff kind: Event lastTimestamp: "2026-07-02T14:39:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:38:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:39:04Z" name: error-404-isvc-57d01-predictor.18be7fdbdf931229 namespace: kserve-ci-e2e-test resourceVersion: "21413" uid: 0852a74c-2021-4766-9a28-1ca9c726e91c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:37:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-57d01 namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: 004182aa-7428-46a8-aca5-9b04ec296fd9 kind: Event lastTimestamp: "2026-07-02T14:38:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:38:32Z" name: error-404-isvc-57d01.18be7fd4e1aa247d namespace: kserve-ci-e2e-test resourceVersion: "20958" uid: cd66362b-7f99-43e8-9b8a-3cb7f6a0a9aa reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-57d01 namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: 004182aa-7428-46a8-aca5-9b04ec296fd9 kind: Event lastTimestamp: "2026-07-02T14:37:34Z" message: 'Failed to update status for InferenceService "error-404-isvc-57d01": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-57d01": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:37:34Z" name: error-404-isvc-57d01.18be7fd4e21c9238 namespace: kserve-ci-e2e-test resourceVersion: "20348" uid: bb7bb1da-beca-4c49-b71c-c45ded7be8b6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-57d01 namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: 004182aa-7428-46a8-aca5-9b04ec296fd9 kind: Event lastTimestamp: "2026-07-02T14:37:34Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-57d01": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:37:34Z" name: error-404-isvc-57d01.18be7fd4e21ce23a namespace: kserve-ci-e2e-test resourceVersion: "20351" uid: 39c2b41b-bb5f-43ee-941e-1e929480bb2f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-57d01 namespace: kserve-ci-e2e-test resourceVersion: "20956" uid: 004182aa-7428-46a8-aca5-9b04ec296fd9 kind: Event lastTimestamp: "2026-07-02T14:38:32Z" message: InferenceService [error-404-isvc-57d01] is Ready metadata: creationTimestamp: "2026-07-02T14:38:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:38:32Z" name: error-404-isvc-57d01.18be7fe2645cd8a3 namespace: kserve-ci-e2e-test resourceVersion: "20957" uid: b7b1038f-45d8-424d-a5b8-d1879dbeb27f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:37:20.318051Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v namespace: kserve-ci-e2e-test resourceVersion: "20055" uid: adc7e05a-8111-434b-8c37-8804fa0c10f4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v to ip-10-0-134-54.ec2.internal metadata: creationTimestamp: "2026-07-02T14:37:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:37:20Z" name: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v.18be7fd183996dfd namespace: kserve-ci-e2e-test resourceVersion: "20059" uid: c179db28-2d3e-451e-bee4-b1a6579e4a75 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:20Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v namespace: kserve-ci-e2e-test resourceVersion: "20057" uid: adc7e05a-8111-434b-8c37-8804fa0c10f4 kind: Event lastTimestamp: "2026-07-02T14:37:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-8937c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:20Z" name: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v.18be7fd18ae90ef9 namespace: kserve-ci-e2e-test resourceVersion: "20082" uid: fd807171-62bd-4d39-9871-6f89c7981681 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:21Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v namespace: kserve-ci-e2e-test resourceVersion: "20063" uid: adc7e05a-8111-434b-8c37-8804fa0c10f4 kind: Event lastTimestamp: "2026-07-02T14:37:21Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:37:21Z" name: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v.18be7fd1c100e1a5 namespace: kserve-ci-e2e-test resourceVersion: "20103" uid: b13385d5-0199-4696-88e0-6649d24f5d61 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v namespace: kserve-ci-e2e-test resourceVersion: "20057" uid: adc7e05a-8111-434b-8c37-8804fa0c10f4 kind: Event lastTimestamp: "2026-07-02T14:37:21Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:21Z" name: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v.18be7fd1c2c71798 namespace: kserve-ci-e2e-test resourceVersion: "20105" uid: e846be25-b408-4d41-b9fd-4717aca82cac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v namespace: kserve-ci-e2e-test resourceVersion: "20057" uid: adc7e05a-8111-434b-8c37-8804fa0c10f4 kind: Event lastTimestamp: "2026-07-02T14:37:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:21Z" name: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v.18be7fd1c76a030a namespace: kserve-ci-e2e-test resourceVersion: "20106" uid: dab75355-659c-45ec-8b20-9566b0dce71f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v namespace: kserve-ci-e2e-test resourceVersion: "20057" uid: adc7e05a-8111-434b-8c37-8804fa0c10f4 kind: Event lastTimestamp: "2026-07-02T14:37:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:21Z" name: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v.18be7fd1c7d87395 namespace: kserve-ci-e2e-test resourceVersion: "20107" uid: 4e34d8df-78ae-47da-8035-461afe9c71dd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v namespace: kserve-ci-e2e-test resourceVersion: "20057" uid: adc7e05a-8111-434b-8c37-8804fa0c10f4 kind: Event lastTimestamp: "2026-07-02T14:37:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:21Z" name: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v.18be7fd1c7e42f61 namespace: kserve-ci-e2e-test resourceVersion: "20108" uid: 6a228870-4275-4c62-abcd-e409c90e8ac7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v namespace: kserve-ci-e2e-test resourceVersion: "20057" uid: adc7e05a-8111-434b-8c37-8804fa0c10f4 kind: Event lastTimestamp: "2026-07-02T14:37:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:21Z" name: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v.18be7fd1cac33b31 namespace: kserve-ci-e2e-test resourceVersion: "20110" uid: 74c90668-1c97-4182-b16c-ce37a0c869ca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v namespace: kserve-ci-e2e-test resourceVersion: "20057" uid: adc7e05a-8111-434b-8c37-8804fa0c10f4 kind: Event lastTimestamp: "2026-07-02T14:37:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:21Z" name: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v.18be7fd1cb3a7601 namespace: kserve-ci-e2e-test resourceVersion: "20111" uid: bca58577-6815-4032-b9d4-232645d563bc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-02T14:37:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v namespace: kserve-ci-e2e-test resourceVersion: "20057" uid: adc7e05a-8111-434b-8c37-8804fa0c10f4 kind: Event lastTimestamp: "2026-07-02T14:38:48Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:48Z" name: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v.18be7fd20e18adec namespace: kserve-ci-e2e-test resourceVersion: "21266" uid: 4f497466-8398-490e-88ef-66d5ea25f27b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v namespace: kserve-ci-e2e-test resourceVersion: "20057" uid: adc7e05a-8111-434b-8c37-8804fa0c10f4 kind: Event lastTimestamp: "2026-07-02T14:38:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:44Z" name: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v.18be7fe521fecac1 namespace: kserve-ci-e2e-test resourceVersion: "21137" uid: 05f3181d-8e1b-42c2-bb8f-7d2c8e6c3fef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v namespace: kserve-ci-e2e-test resourceVersion: "20057" uid: adc7e05a-8111-434b-8c37-8804fa0c10f4 kind: Event lastTimestamp: "2026-07-02T14:38:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:44Z" name: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v.18be7fe521ff9c14 namespace: kserve-ci-e2e-test resourceVersion: "21141" uid: 166c846d-9b49-401f-bf88-0e5e2a5781c6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v namespace: kserve-ci-e2e-test resourceVersion: "20057" uid: adc7e05a-8111-434b-8c37-8804fa0c10f4 kind: Event lastTimestamp: "2026-07-02T14:38:48Z" message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp 10.133.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:38:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:48Z" name: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v.18be7fe61451ef45 namespace: kserve-ci-e2e-test resourceVersion: "21265" uid: 1b1cec13-6a04-40c0-b0a0-eff008390ecd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8937c-predictor-f4ccb5bcf namespace: kserve-ci-e2e-test resourceVersion: "20046" uid: 8b74a7a5-f9bd-4942-b0f4-9be7a0528012 kind: Event lastTimestamp: "2026-07-02T14:37:20Z" message: 'Created pod: error-404-isvc-8937c-predictor-f4ccb5bcf-n2l7v' metadata: creationTimestamp: "2026-07-02T14:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:37:20Z" name: error-404-isvc-8937c-predictor-f4ccb5bcf.18be7fd18328de5e namespace: kserve-ci-e2e-test resourceVersion: "20056" uid: 4d029203-d124-43d0-ac28-ca2650610a7b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8937c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20043" uid: 36149e98-d84d-4ac7-899d-de3140fd77ea kind: Event lastTimestamp: "2026-07-02T14:37:20Z" message: Scaled up replica set error-404-isvc-8937c-predictor-f4ccb5bcf from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:37:20Z" name: error-404-isvc-8937c-predictor.18be7fd1816f9a4c namespace: kserve-ci-e2e-test resourceVersion: "20049" uid: b26ff862-bdbc-4eee-aedc-9b5e9ab4bcef reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:37:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8937c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20045" uid: eba236b2-64ce-42d7-a77c-c1cc2ecad67c kind: Event lastTimestamp: "2026-07-02T14:37:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:37:50Z" name: error-404-isvc-8937c-predictor.18be7fd4ffe8297e namespace: kserve-ci-e2e-test resourceVersion: "20558" uid: 53cfd290-dcd3-4fc3-9bd9-80389fcd54a1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:37:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8937c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20045" uid: eba236b2-64ce-42d7-a77c-c1cc2ecad67c kind: Event lastTimestamp: "2026-07-02T14:37:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:37:50Z" name: error-404-isvc-8937c-predictor.18be7fd5005cd490 namespace: kserve-ci-e2e-test resourceVersion: "20559" uid: ddd6676e-8984-480a-a5cc-10378893dffa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:38:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8937c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20397" uid: eba236b2-64ce-42d7-a77c-c1cc2ecad67c kind: Event lastTimestamp: "2026-07-02T14:38:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:38:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:35Z" name: error-404-isvc-8937c-predictor.18be7fdbfde76ebf namespace: kserve-ci-e2e-test resourceVersion: "21018" uid: 227b45f8-f083-4f1b-a556-5019c6c764ee reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:38:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8937c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20397" uid: eba236b2-64ce-42d7-a77c-c1cc2ecad67c kind: Event lastTimestamp: "2026-07-02T14:38:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:38:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:35Z" name: error-404-isvc-8937c-predictor.18be7fdbfe586bc5 namespace: kserve-ci-e2e-test resourceVersion: "21019" uid: 72f59077-1126-450a-8194-ecd90e1e9376 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:37:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8937c namespace: kserve-ci-e2e-test resourceVersion: "20025" uid: 7b63064d-d932-413f-8f4e-51e241ce7507 kind: Event lastTimestamp: "2026-07-02T14:38:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:38:18Z" name: error-404-isvc-8937c.18be7fd181a06fbb namespace: kserve-ci-e2e-test resourceVersion: "20797" uid: 706a07b6-7084-4476-83f7-cc458ab8d494 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8937c namespace: kserve-ci-e2e-test resourceVersion: "20025" uid: 7b63064d-d932-413f-8f4e-51e241ce7507 kind: Event lastTimestamp: "2026-07-02T14:37:20Z" message: 'Failed to update status for InferenceService "error-404-isvc-8937c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8937c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:37:20Z" name: error-404-isvc-8937c.18be7fd181fa942a namespace: kserve-ci-e2e-test resourceVersion: "20052" uid: 621c56d0-0bfd-4244-a7c3-cd3d497f84b2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8937c namespace: kserve-ci-e2e-test resourceVersion: "20025" uid: 7b63064d-d932-413f-8f4e-51e241ce7507 kind: Event lastTimestamp: "2026-07-02T14:37:20Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8937c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:37:20Z" name: error-404-isvc-8937c.18be7fd181fada88 namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: c6b5d485-c513-4c4d-8119-aff1ccf08600 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8937c namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: 7b63064d-d932-413f-8f4e-51e241ce7507 kind: Event lastTimestamp: "2026-07-02T14:38:18Z" message: InferenceService [error-404-isvc-8937c] is Ready metadata: creationTimestamp: "2026-07-02T14:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:38:18Z" name: error-404-isvc-8937c.18be7fdf21a8488f namespace: kserve-ci-e2e-test resourceVersion: "20795" uid: d134682e-8a6a-4f95-b134-f9f42b22390b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:28:05.563930Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9a7ca-predictor-c544cc7fc-jks4s namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: 7d71ebfa-3c34-40a6-9373-36a7a7a9a19c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-9a7ca-predictor-c544cc7fc-jks4s to ip-10-0-134-54.ec2.internal metadata: creationTimestamp: "2026-07-02T14:28:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:28:05Z" name: error-404-isvc-9a7ca-predictor-c544cc7fc-jks4s.18be7f5059ad50db namespace: kserve-ci-e2e-test resourceVersion: "15777" uid: 75a35d60-ae1a-43c0-bfff-71b5ffab35f4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:05Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9a7ca-predictor-c544cc7fc-jks4s namespace: kserve-ci-e2e-test resourceVersion: "15780" uid: 7d71ebfa-3c34-40a6-9373-36a7a7a9a19c kind: Event lastTimestamp: "2026-07-02T14:28:05Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:28:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:28:06Z" name: error-404-isvc-9a7ca-predictor-c544cc7fc-jks4s.18be7f5073a0b235 namespace: kserve-ci-e2e-test resourceVersion: "15804" uid: 6bf94923-ead4-4b47-9aa1-e8c3e0bd64d3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9a7ca-predictor-c544cc7fc-jks4s namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 7d71ebfa-3c34-40a6-9373-36a7a7a9a19c kind: Event lastTimestamp: "2026-07-02T14:28:06Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:28:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:06Z" name: error-404-isvc-9a7ca-predictor-c544cc7fc-jks4s.18be7f507541cc2f namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: ffd7cd7c-68ca-44d0-be7a-1e4504150e14 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9a7ca-predictor-c544cc7fc-jks4s namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 7d71ebfa-3c34-40a6-9373-36a7a7a9a19c kind: Event lastTimestamp: "2026-07-02T14:28:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:28:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:06Z" name: error-404-isvc-9a7ca-predictor-c544cc7fc-jks4s.18be7f5078425cb9 namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: 60cb78fa-7a05-4702-92dd-aeeddba3f1b9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9a7ca-predictor-c544cc7fc-jks4s namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 7d71ebfa-3c34-40a6-9373-36a7a7a9a19c kind: Event lastTimestamp: "2026-07-02T14:28:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:28:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:06Z" name: error-404-isvc-9a7ca-predictor-c544cc7fc-jks4s.18be7f5078d09924 namespace: kserve-ci-e2e-test resourceVersion: "15810" uid: 11dfc522-ffe5-48cc-863f-39a7ca33f019 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9a7ca-predictor-c544cc7fc-jks4s namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 7d71ebfa-3c34-40a6-9373-36a7a7a9a19c kind: Event lastTimestamp: "2026-07-02T14:28:06Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:28:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:06Z" name: error-404-isvc-9a7ca-predictor-c544cc7fc-jks4s.18be7f5078dc1097 namespace: kserve-ci-e2e-test resourceVersion: "15811" uid: a8d69726-60e5-44e8-9a1f-fa937acefb21 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9a7ca-predictor-c544cc7fc-jks4s namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 7d71ebfa-3c34-40a6-9373-36a7a7a9a19c kind: Event lastTimestamp: "2026-07-02T14:28:06Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:28:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:06Z" name: error-404-isvc-9a7ca-predictor-c544cc7fc-jks4s.18be7f507b8e31ce namespace: kserve-ci-e2e-test resourceVersion: "15812" uid: f29be078-f0df-4baa-be04-5b266e741869 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9a7ca-predictor-c544cc7fc-jks4s namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 7d71ebfa-3c34-40a6-9373-36a7a7a9a19c kind: Event lastTimestamp: "2026-07-02T14:28:06Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:28:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:06Z" name: error-404-isvc-9a7ca-predictor-c544cc7fc-jks4s.18be7f507c029b6b namespace: kserve-ci-e2e-test resourceVersion: "15813" uid: 0b2a50c0-091f-4b44-8a96-5d370a62c8a9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T14:28:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9a7ca-predictor-c544cc7fc-jks4s namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 7d71ebfa-3c34-40a6-9373-36a7a7a9a19c kind: Event lastTimestamp: "2026-07-02T14:37:22Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:28:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:22Z" name: error-404-isvc-9a7ca-predictor-c544cc7fc-jks4s.18be7f508eb570e9 namespace: kserve-ci-e2e-test resourceVersion: "20125" uid: 8354e5a2-fdc5-409d-9d23-ec979d39fbb4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9a7ca-predictor-c544cc7fc-jks4s namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 7d71ebfa-3c34-40a6-9373-36a7a7a9a19c kind: Event lastTimestamp: "2026-07-02T14:37:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:20Z" name: error-404-isvc-9a7ca-predictor-c544cc7fc-jks4s.18be7fd1888a81d8 namespace: kserve-ci-e2e-test resourceVersion: "20078" uid: d28f9b0a-ff15-44ce-add0-6f34e610ee3a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9a7ca-predictor-c544cc7fc-jks4s namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 7d71ebfa-3c34-40a6-9373-36a7a7a9a19c kind: Event lastTimestamp: "2026-07-02T14:37:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:20Z" name: error-404-isvc-9a7ca-predictor-c544cc7fc-jks4s.18be7fd1888b3f8f namespace: kserve-ci-e2e-test resourceVersion: "20079" uid: da3ed7c8-4fbb-476a-80e6-18fdb787b5cf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9a7ca-predictor-c544cc7fc-jks4s namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 7d71ebfa-3c34-40a6-9373-36a7a7a9a19c kind: Event lastTimestamp: "2026-07-02T14:37:22Z" message: 'Readiness probe failed: Get "https://10.133.0.27:8643/healthz": dial tcp 10.133.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:22Z" name: error-404-isvc-9a7ca-predictor-c544cc7fc-jks4s.18be7fd2032c178f namespace: kserve-ci-e2e-test resourceVersion: "20124" uid: 8b772ee6-46c0-4595-a2f7-eaa9650089f1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-9a7ca-predictor-c544cc7fc namespace: kserve-ci-e2e-test resourceVersion: "15763" uid: 1736a3e9-47e9-48c3-8cd9-03da92c9543a kind: Event lastTimestamp: "2026-07-02T14:28:05Z" message: 'Created pod: error-404-isvc-9a7ca-predictor-c544cc7fc-jks4s' metadata: creationTimestamp: "2026-07-02T14:28:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:05Z" name: error-404-isvc-9a7ca-predictor-c544cc7fc.18be7f5058cce664 namespace: kserve-ci-e2e-test resourceVersion: "15774" uid: 59577446-84ef-43ee-9263-6c1e5576d417 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-9a7ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "15760" uid: 6f165cfc-bc29-40ed-b72e-d58d8fca232e kind: Event lastTimestamp: "2026-07-02T14:28:05Z" message: Scaled up replica set error-404-isvc-9a7ca-predictor-c544cc7fc from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:28:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:05Z" name: error-404-isvc-9a7ca-predictor.18be7f5056a3c95b namespace: kserve-ci-e2e-test resourceVersion: "15766" uid: ae48a9f3-9eb9-4e10-b69c-8733e2f47b90 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9a7ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "15761" uid: 498fba0d-538e-49f3-9ce4-3daa1b0ba810 kind: Event lastTimestamp: "2026-07-02T14:28:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:28:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:20Z" name: error-404-isvc-9a7ca-predictor.18be7f53d511be18 namespace: kserve-ci-e2e-test resourceVersion: "15995" uid: 4d476e23-fb71-4d2a-b307-3823e9a65706 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9a7ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "15761" uid: 498fba0d-538e-49f3-9ce4-3daa1b0ba810 kind: Event lastTimestamp: "2026-07-02T14:28:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:28:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:20Z" name: error-404-isvc-9a7ca-predictor.18be7f53d5898f1d namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: cd56ed95-a7e2-4a88-b8fc-d4d6248511ec reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:28:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9a7ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "15996" uid: 498fba0d-538e-49f3-9ce4-3daa1b0ba810 kind: Event lastTimestamp: "2026-07-02T14:29:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:28:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:29:05Z" name: error-404-isvc-9a7ca-predictor.18be7f575436ddc1 namespace: kserve-ci-e2e-test resourceVersion: "16389" uid: d192fddd-2953-466c-bbaa-6431fef00d76 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:28:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9a7ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "15996" uid: 498fba0d-538e-49f3-9ce4-3daa1b0ba810 kind: Event lastTimestamp: "2026-07-02T14:29:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:28:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:29:05Z" name: error-404-isvc-9a7ca-predictor.18be7f5754a0349d namespace: kserve-ci-e2e-test resourceVersion: "16391" uid: 5164792a-9cae-4ac4-a6c8-63d86d20f846 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:28:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9a7ca namespace: kserve-ci-e2e-test resourceVersion: "15742" uid: 6e31d6b1-eb56-43b2-b0e8-274a063bf0f3 kind: Event lastTimestamp: "2026-07-02T14:28:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:28:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:28:52Z" name: error-404-isvc-9a7ca.18be7f5056a90e23 namespace: kserve-ci-e2e-test resourceVersion: "16257" uid: ddba9548-d051-4a44-991f-46732112c94e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9a7ca namespace: kserve-ci-e2e-test resourceVersion: "15742" uid: 6e31d6b1-eb56-43b2-b0e8-274a063bf0f3 kind: Event lastTimestamp: "2026-07-02T14:28:05Z" message: 'Failed to update status for InferenceService "error-404-isvc-9a7ca": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9a7ca": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:28:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:28:05Z" name: error-404-isvc-9a7ca.18be7f50572212c1 namespace: kserve-ci-e2e-test resourceVersion: "15770" uid: 9ff4e3e6-fecd-49ff-aa4c-30a889c6d347 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9a7ca namespace: kserve-ci-e2e-test resourceVersion: "15742" uid: 6e31d6b1-eb56-43b2-b0e8-274a063bf0f3 kind: Event lastTimestamp: "2026-07-02T14:28:05Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9a7ca": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:28:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:28:05Z" name: error-404-isvc-9a7ca.18be7f5057227210 namespace: kserve-ci-e2e-test resourceVersion: "15771" uid: 2f417b3b-a530-450e-9977-79046b290dfe reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9a7ca namespace: kserve-ci-e2e-test resourceVersion: "16255" uid: 6e31d6b1-eb56-43b2-b0e8-274a063bf0f3 kind: Event lastTimestamp: "2026-07-02T14:28:52Z" message: InferenceService [error-404-isvc-9a7ca] is Ready metadata: creationTimestamp: "2026-07-02T14:28:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:28:52Z" name: error-404-isvc-9a7ca.18be7f5b4ac6dc84 namespace: kserve-ci-e2e-test resourceVersion: "16256" uid: 99638a26-c96d-4b5a-828e-2413e210b4b1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:38:44.753000Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm namespace: kserve-ci-e2e-test resourceVersion: "21188" uid: b869f464-c7f0-4c3a-9a25-29730c80d07b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm to ip-10-0-134-54.ec2.internal metadata: creationTimestamp: "2026-07-02T14:38:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:38:44Z" name: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm.18be7fe52c507ef1 namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: 28ecc153-7f08-444a-87eb-a5f404e21d13 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:44Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm namespace: kserve-ci-e2e-test resourceVersion: "21190" uid: b869f464-c7f0-4c3a-9a25-29730c80d07b kind: Event lastTimestamp: "2026-07-02T14:38:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-c3f32-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:44Z" name: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm.18be7fe536044a6d namespace: kserve-ci-e2e-test resourceVersion: "21204" uid: 5a40822f-f34e-4412-b7ed-79199a8004bf reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:45Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm namespace: kserve-ci-e2e-test resourceVersion: "21196" uid: b869f464-c7f0-4c3a-9a25-29730c80d07b kind: Event lastTimestamp: "2026-07-02T14:38:45Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:38:45Z" name: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm.18be7fe56ac09c4b namespace: kserve-ci-e2e-test resourceVersion: "21224" uid: 6628a047-f642-44f7-a4e7-cfc0bd2752fd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm namespace: kserve-ci-e2e-test resourceVersion: "21190" uid: b869f464-c7f0-4c3a-9a25-29730c80d07b kind: Event lastTimestamp: "2026-07-02T14:38:45Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:45Z" name: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm.18be7fe56cc9253b namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: e0fe17b3-b1da-415d-886b-80a1da3dbb96 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm namespace: kserve-ci-e2e-test resourceVersion: "21190" uid: b869f464-c7f0-4c3a-9a25-29730c80d07b kind: Event lastTimestamp: "2026-07-02T14:38:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:45Z" name: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm.18be7fe56fa17555 namespace: kserve-ci-e2e-test resourceVersion: "21231" uid: e3d7e1f2-6655-45e5-839b-680f6e7a7431 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm namespace: kserve-ci-e2e-test resourceVersion: "21190" uid: b869f464-c7f0-4c3a-9a25-29730c80d07b kind: Event lastTimestamp: "2026-07-02T14:38:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:45Z" name: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm.18be7fe5701a03b3 namespace: kserve-ci-e2e-test resourceVersion: "21232" uid: ed25f112-3634-43b9-a0c8-64890f2c0c55 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm namespace: kserve-ci-e2e-test resourceVersion: "21190" uid: b869f464-c7f0-4c3a-9a25-29730c80d07b kind: Event lastTimestamp: "2026-07-02T14:38:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:45Z" name: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm.18be7fe57026189b namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: 1628e5a2-9953-4955-830e-dcb61c727c21 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm namespace: kserve-ci-e2e-test resourceVersion: "21190" uid: b869f464-c7f0-4c3a-9a25-29730c80d07b kind: Event lastTimestamp: "2026-07-02T14:38:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:45Z" name: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm.18be7fe57330b2a0 namespace: kserve-ci-e2e-test resourceVersion: "21234" uid: fd5baa80-17e8-41ec-a42f-1a005df8a207 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm namespace: kserve-ci-e2e-test resourceVersion: "21190" uid: b869f464-c7f0-4c3a-9a25-29730c80d07b kind: Event lastTimestamp: "2026-07-02T14:38:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:45Z" name: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm.18be7fe573c1a6b3 namespace: kserve-ci-e2e-test resourceVersion: "21235" uid: f477eb36-02d6-4de8-9cb2-ab6516118f68 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T14:38:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm namespace: kserve-ci-e2e-test resourceVersion: "21190" uid: b869f464-c7f0-4c3a-9a25-29730c80d07b kind: Event lastTimestamp: "2026-07-02T14:48:03Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:38:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:03Z" name: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm.18be7fe5b32ba32c namespace: kserve-ci-e2e-test resourceVersion: "25708" uid: 8055f1a6-1379-49ea-bb3e-f8f6915cc74f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm namespace: kserve-ci-e2e-test resourceVersion: "21190" uid: b869f464-c7f0-4c3a-9a25-29730c80d07b kind: Event lastTimestamp: "2026-07-02T14:47:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:47:59Z" name: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm.18be80664c1bc17b namespace: kserve-ci-e2e-test resourceVersion: "25434" uid: c8130d7a-9f6d-461e-8317-5d65ea3f5124 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm namespace: kserve-ci-e2e-test resourceVersion: "21190" uid: b869f464-c7f0-4c3a-9a25-29730c80d07b kind: Event lastTimestamp: "2026-07-02T14:47:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:47:59Z" name: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm.18be80664c1c4dd2 namespace: kserve-ci-e2e-test resourceVersion: "25437" uid: 97789977-4e94-4527-8fcb-5914c4fbe135 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm namespace: kserve-ci-e2e-test resourceVersion: "21190" uid: b869f464-c7f0-4c3a-9a25-29730c80d07b kind: Event lastTimestamp: "2026-07-02T14:48:03Z" message: 'Readiness probe failed: Get "https://10.133.0.35:8643/healthz": dial tcp 10.133.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:48:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:03Z" name: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm.18be80672787f836 namespace: kserve-ci-e2e-test resourceVersion: "25707" uid: 85278dca-94c3-45d1-ba4d-f795bc101eef reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c3f32-predictor-5c75d8fd6f namespace: kserve-ci-e2e-test resourceVersion: "21178" uid: afe4818a-0fe1-4508-be09-ce9288dc39ed kind: Event lastTimestamp: "2026-07-02T14:38:44Z" message: 'Created pod: error-404-isvc-c3f32-predictor-5c75d8fd6f-54vnm' metadata: creationTimestamp: "2026-07-02T14:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:44Z" name: error-404-isvc-c3f32-predictor-5c75d8fd6f.18be7fe52be24950 namespace: kserve-ci-e2e-test resourceVersion: "21189" uid: 8f7c33e8-66a8-46a3-9e2a-56d33349f1b3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c3f32-predictor namespace: kserve-ci-e2e-test resourceVersion: "21175" uid: edeb56c2-7e47-4479-b9c5-282bca101e81 kind: Event lastTimestamp: "2026-07-02T14:38:44Z" message: Scaled up replica set error-404-isvc-c3f32-predictor-5c75d8fd6f from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:44Z" name: error-404-isvc-c3f32-predictor.18be7fe52a1ef0ce namespace: kserve-ci-e2e-test resourceVersion: "21180" uid: ebdc7157-a58f-4afb-8225-97433a4368f9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:38:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3f32-predictor namespace: kserve-ci-e2e-test resourceVersion: "21177" uid: 60fd0f9c-6551-4e0a-aa0f-cd0556bafdce kind: Event lastTimestamp: "2026-07-02T14:39:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:38:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:39:29Z" name: error-404-isvc-c3f32-predictor.18be7fe8a8afb223 namespace: kserve-ci-e2e-test resourceVersion: "21596" uid: f1c2610d-26a0-42e5-af90-70a9aa7b38bf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:38:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3f32-predictor namespace: kserve-ci-e2e-test resourceVersion: "21177" uid: 60fd0f9c-6551-4e0a-aa0f-cd0556bafdce kind: Event lastTimestamp: "2026-07-02T14:39:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:38:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:39:29Z" name: error-404-isvc-c3f32-predictor.18be7fe8a9215d77 namespace: kserve-ci-e2e-test resourceVersion: "21597" uid: 0de606fb-4f95-4fd5-a512-8e0c7be89b24 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:39:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3f32-predictor namespace: kserve-ci-e2e-test resourceVersion: "21367" uid: 60fd0f9c-6551-4e0a-aa0f-cd0556bafdce kind: Event lastTimestamp: "2026-07-02T14:39:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:39:59Z" name: error-404-isvc-c3f32-predictor.18be7ff325159210 namespace: kserve-ci-e2e-test resourceVersion: "21866" uid: 9762b0e1-5581-453f-91a7-ad598004f5f8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:39:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3f32-predictor namespace: kserve-ci-e2e-test resourceVersion: "21367" uid: 60fd0f9c-6551-4e0a-aa0f-cd0556bafdce kind: Event lastTimestamp: "2026-07-02T14:39:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:39:59Z" name: error-404-isvc-c3f32-predictor.18be7ff325939216 namespace: kserve-ci-e2e-test resourceVersion: "21867" uid: 50929311-d36a-41d2-ad76-1d16226e0279 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:38:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3f32 namespace: kserve-ci-e2e-test resourceVersion: "21149" uid: 946b3831-6392-42fe-b28d-7d1ec37220a4 kind: Event lastTimestamp: "2026-07-02T14:39:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:39:33Z" name: error-404-isvc-c3f32.18be7fe52a64f5cd namespace: kserve-ci-e2e-test resourceVersion: "21637" uid: 784a6d9d-18db-4039-9f19-17a6649e3ee5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3f32 namespace: kserve-ci-e2e-test resourceVersion: "21149" uid: 946b3831-6392-42fe-b28d-7d1ec37220a4 kind: Event lastTimestamp: "2026-07-02T14:38:44Z" message: 'Failed to update status for InferenceService "error-404-isvc-c3f32": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c3f32": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:38:44Z" name: error-404-isvc-c3f32.18be7fe52ae56375 namespace: kserve-ci-e2e-test resourceVersion: "21185" uid: 7b98a867-51b8-4ddf-9bd9-a5eab7cdebf9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3f32 namespace: kserve-ci-e2e-test resourceVersion: "21149" uid: 946b3831-6392-42fe-b28d-7d1ec37220a4 kind: Event lastTimestamp: "2026-07-02T14:38:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c3f32": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:38:44Z" name: error-404-isvc-c3f32.18be7fe52ae5b7eb namespace: kserve-ci-e2e-test resourceVersion: "21187" uid: 4777fb61-7443-4ae8-91ea-2db6873cb932 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:39:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3f32 namespace: kserve-ci-e2e-test resourceVersion: "21635" uid: 946b3831-6392-42fe-b28d-7d1ec37220a4 kind: Event lastTimestamp: "2026-07-02T14:39:33Z" message: InferenceService [error-404-isvc-c3f32] is Ready metadata: creationTimestamp: "2026-07-02T14:39:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:39:33Z" name: error-404-isvc-c3f32.18be7ff06ebc4c1e namespace: kserve-ci-e2e-test resourceVersion: "21636" uid: 4ac90bc3-4c78-45c5-9240-8ec63831c341 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:47:59.506279Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cb06c-predictor-7df7f44c4c-5bh47 namespace: kserve-ci-e2e-test resourceVersion: "25491" uid: 467d3d03-2a76-4f21-af2d-7f0eb0ee6a3c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-cb06c-predictor-7df7f44c4c-5bh47 to ip-10-0-134-54.ec2.internal metadata: creationTimestamp: "2026-07-02T14:47:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:47:59Z" name: error-404-isvc-cb06c-predictor-7df7f44c4c-5bh47.18be8066562fbfe2 namespace: kserve-ci-e2e-test resourceVersion: "25495" uid: e5abd4d0-f8f7-475d-854c-7fc80658d8e9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:59Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cb06c-predictor-7df7f44c4c-5bh47 namespace: kserve-ci-e2e-test resourceVersion: "25499" uid: 467d3d03-2a76-4f21-af2d-7f0eb0ee6a3c kind: Event lastTimestamp: "2026-07-02T14:47:59Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:47:59Z" name: error-404-isvc-cb06c-predictor-7df7f44c4c-5bh47.18be806672d85caf namespace: kserve-ci-e2e-test resourceVersion: "25592" uid: c9e07f80-3caf-4f01-9e48-c902e6f4e16c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cb06c-predictor-7df7f44c4c-5bh47 namespace: kserve-ci-e2e-test resourceVersion: "25493" uid: 467d3d03-2a76-4f21-af2d-7f0eb0ee6a3c kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:00Z" name: error-404-isvc-cb06c-predictor-7df7f44c4c-5bh47.18be8066768c3b86 namespace: kserve-ci-e2e-test resourceVersion: "25611" uid: 845930c7-b78b-48b4-b9f8-858f55eaf0b1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cb06c-predictor-7df7f44c4c-5bh47 namespace: kserve-ci-e2e-test resourceVersion: "25493" uid: 467d3d03-2a76-4f21-af2d-7f0eb0ee6a3c kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:00Z" name: error-404-isvc-cb06c-predictor-7df7f44c4c-5bh47.18be80667e08c511 namespace: kserve-ci-e2e-test resourceVersion: "25628" uid: b10fd72e-8c57-4bd3-83ad-5a6333006064 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cb06c-predictor-7df7f44c4c-5bh47 namespace: kserve-ci-e2e-test resourceVersion: "25493" uid: 467d3d03-2a76-4f21-af2d-7f0eb0ee6a3c kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:00Z" name: error-404-isvc-cb06c-predictor-7df7f44c4c-5bh47.18be80667e81faf9 namespace: kserve-ci-e2e-test resourceVersion: "25629" uid: 945d9003-3853-407f-aedd-b1e6f20035cb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cb06c-predictor-7df7f44c4c-5bh47 namespace: kserve-ci-e2e-test resourceVersion: "25493" uid: 467d3d03-2a76-4f21-af2d-7f0eb0ee6a3c kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:00Z" name: error-404-isvc-cb06c-predictor-7df7f44c4c-5bh47.18be80667e8df848 namespace: kserve-ci-e2e-test resourceVersion: "25630" uid: 9282ee04-e751-4315-b8f8-2d536044ef21 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cb06c-predictor-7df7f44c4c-5bh47 namespace: kserve-ci-e2e-test resourceVersion: "25493" uid: 467d3d03-2a76-4f21-af2d-7f0eb0ee6a3c kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:00Z" name: error-404-isvc-cb06c-predictor-7df7f44c4c-5bh47.18be806681f5ba8d namespace: kserve-ci-e2e-test resourceVersion: "25632" uid: 8545a40c-442a-444b-815c-55f8775a1474 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cb06c-predictor-7df7f44c4c-5bh47 namespace: kserve-ci-e2e-test resourceVersion: "25493" uid: 467d3d03-2a76-4f21-af2d-7f0eb0ee6a3c kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:00Z" name: error-404-isvc-cb06c-predictor-7df7f44c4c-5bh47.18be8066828de30e namespace: kserve-ci-e2e-test resourceVersion: "25634" uid: 36308b04-0923-42b2-82cb-7e6ed320be8a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T14:48:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cb06c-predictor-7df7f44c4c-5bh47 namespace: kserve-ci-e2e-test resourceVersion: "25493" uid: 467d3d03-2a76-4f21-af2d-7f0eb0ee6a3c kind: Event lastTimestamp: "2026-07-02T14:48:47Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:47Z" name: error-404-isvc-cb06c-predictor-7df7f44c4c-5bh47.18be8066b97d58bc namespace: kserve-ci-e2e-test resourceVersion: "26381" uid: eacaddac-abc7-44cd-996b-78fa2f61848f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T15:05:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cb06c-predictor-7df7f44c4c-5bh47 namespace: kserve-ci-e2e-test resourceVersion: "25493" uid: 467d3d03-2a76-4f21-af2d-7f0eb0ee6a3c kind: Event lastTimestamp: "2026-07-02T15:05:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T15:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T15:05:29Z" name: error-404-isvc-cb06c-predictor-7df7f44c4c-5bh47.18be815ab4a8f6a8 namespace: kserve-ci-e2e-test resourceVersion: "33525" uid: 6782f98b-2ba7-490b-9d13-62435f5958da reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T15:05:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cb06c-predictor-7df7f44c4c-5bh47 namespace: kserve-ci-e2e-test resourceVersion: "25493" uid: 467d3d03-2a76-4f21-af2d-7f0eb0ee6a3c kind: Event lastTimestamp: "2026-07-02T15:05:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T15:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T15:05:29Z" name: error-404-isvc-cb06c-predictor-7df7f44c4c-5bh47.18be815ab4a978f9 namespace: kserve-ci-e2e-test resourceVersion: "33526" uid: ad42dc47-c765-4973-9c72-d1312cac2aaa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-cb06c-predictor-7df7f44c4c namespace: kserve-ci-e2e-test resourceVersion: "25483" uid: 56028de1-bde7-4934-aee2-681900ec9019 kind: Event lastTimestamp: "2026-07-02T14:47:59Z" message: 'Created pod: error-404-isvc-cb06c-predictor-7df7f44c4c-5bh47' metadata: creationTimestamp: "2026-07-02T14:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:47:59Z" name: error-404-isvc-cb06c-predictor-7df7f44c4c.18be806655a11927 namespace: kserve-ci-e2e-test resourceVersion: "25492" uid: f97782da-3f24-400b-80b0-b497139e8ac1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-cb06c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25480" uid: e7b335ba-aada-4766-b891-0b3b25f7b102 kind: Event lastTimestamp: "2026-07-02T14:47:59Z" message: Scaled up replica set error-404-isvc-cb06c-predictor-7df7f44c4c from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:47:59Z" name: error-404-isvc-cb06c-predictor.18be80665404cb7e namespace: kserve-ci-e2e-test resourceVersion: "25484" uid: 4eba3630-6985-4a84-ab3c-bc1941a06074 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cb06c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25482" uid: 87290f51-d7ae-4640-9de5-79b939f4badd kind: Event lastTimestamp: "2026-07-02T14:48:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:48:14Z" name: error-404-isvc-cb06c-predictor.18be8069d2a02f68 namespace: kserve-ci-e2e-test resourceVersion: "25820" uid: 4ae209f7-053f-4b9a-ac84-d8d916211d30 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cb06c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25482" uid: 87290f51-d7ae-4640-9de5-79b939f4badd kind: Event lastTimestamp: "2026-07-02T14:48:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:48:14Z" name: error-404-isvc-cb06c-predictor.18be8069d30d4760 namespace: kserve-ci-e2e-test resourceVersion: "25821" uid: 3bbda85c-4357-454d-9873-6869305349d6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:48:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cb06c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25819" uid: 87290f51-d7ae-4640-9de5-79b939f4badd kind: Event lastTimestamp: "2026-07-02T14:49:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:49:29Z" name: error-404-isvc-cb06c-predictor.18be806d51af580e namespace: kserve-ci-e2e-test resourceVersion: "26824" uid: 6ecc0582-ac19-43e2-a86e-c997bd7e1d77 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:48:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cb06c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25819" uid: 87290f51-d7ae-4640-9de5-79b939f4badd kind: Event lastTimestamp: "2026-07-02T14:49:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:49:29Z" name: error-404-isvc-cb06c-predictor.18be806d521b7f5a namespace: kserve-ci-e2e-test resourceVersion: "26825" uid: bacaa97d-6067-40aa-9b99-afb30264c823 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:47:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cb06c namespace: kserve-ci-e2e-test resourceVersion: "25466" uid: 928e243b-319e-4387-9479-9f09e191c74d kind: Event lastTimestamp: "2026-07-02T14:48:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:48:57Z" name: error-404-isvc-cb06c.18be80665416a0d7 namespace: kserve-ci-e2e-test resourceVersion: "26495" uid: 155198e9-e961-4c58-93c1-8f45df181348 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cb06c namespace: kserve-ci-e2e-test resourceVersion: "25466" uid: 928e243b-319e-4387-9479-9f09e191c74d kind: Event lastTimestamp: "2026-07-02T14:47:59Z" message: 'Failed to update status for InferenceService "error-404-isvc-cb06c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-cb06c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:47:59Z" name: error-404-isvc-cb06c.18be806654afa86e namespace: kserve-ci-e2e-test resourceVersion: "25488" uid: 0654d656-e18a-47e1-ac1f-8eeffc558b69 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cb06c namespace: kserve-ci-e2e-test resourceVersion: "25466" uid: 928e243b-319e-4387-9479-9f09e191c74d kind: Event lastTimestamp: "2026-07-02T14:47:59Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-cb06c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:47:59Z" name: error-404-isvc-cb06c.18be806654aff110 namespace: kserve-ci-e2e-test resourceVersion: "25490" uid: 79946a52-eba7-4b25-9ff6-c069726eac4d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cb06c namespace: kserve-ci-e2e-test resourceVersion: "26488" uid: 928e243b-319e-4387-9479-9f09e191c74d kind: Event lastTimestamp: "2026-07-02T14:48:57Z" message: InferenceService [error-404-isvc-cb06c] is Ready metadata: creationTimestamp: "2026-07-02T14:48:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:48:57Z" name: error-404-isvc-cb06c.18be8073d45f11ca namespace: kserve-ci-e2e-test resourceVersion: "26489" uid: 90c19dd7-97e4-44a6-b92c-d31a36127abf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:25:35.807026Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck namespace: kserve-ci-e2e-test resourceVersion: "14050" uid: e3a1b01f-198d-4301-b0fe-388cdbbbdd46 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-df2c1-predictor-688b4ccf47-fsdck to ip-10-0-134-54.ec2.internal metadata: creationTimestamp: "2026-07-02T14:25:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:25:35Z" name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck.18be7f2d7b7853ac namespace: kserve-ci-e2e-test resourceVersion: "14054" uid: aed3805b-b041-48e1-b060-5944a2ed80bc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:36Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck namespace: kserve-ci-e2e-test resourceVersion: "14060" uid: e3a1b01f-198d-4301-b0fe-388cdbbbdd46 kind: Event lastTimestamp: "2026-07-02T14:25:36Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:25:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:25:36Z" name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck.18be7f2d955e716d namespace: kserve-ci-e2e-test resourceVersion: "14090" uid: ee78734e-e216-4292-887f-762e233c429c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck namespace: kserve-ci-e2e-test resourceVersion: "14052" uid: e3a1b01f-198d-4301-b0fe-388cdbbbdd46 kind: Event lastTimestamp: "2026-07-02T14:25:36Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1697" metadata: creationTimestamp: "2026-07-02T14:25:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:36Z" name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck.18be7f2d96d3e87b namespace: kserve-ci-e2e-test resourceVersion: "14096" uid: 97d129cf-12f6-4b20-8e6d-5375962683be reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck namespace: kserve-ci-e2e-test resourceVersion: "14052" uid: e3a1b01f-198d-4301-b0fe-388cdbbbdd46 kind: Event lastTimestamp: "2026-07-02T14:25:51Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1697" in 15.674s (15.674s including waiting). Image size: 1334785147 bytes.' metadata: creationTimestamp: "2026-07-02T14:25:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:51Z" name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck.18be7f313d1ce3e2 namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 9e744227-0bfe-47f5-80b4-4f010b646d23 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck namespace: kserve-ci-e2e-test resourceVersion: "14052" uid: e3a1b01f-198d-4301-b0fe-388cdbbbdd46 kind: Event lastTimestamp: "2026-07-02T14:25:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:25:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:52Z" name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck.18be7f31409a8abd namespace: kserve-ci-e2e-test resourceVersion: "14259" uid: 29cc2a5b-80e4-486c-8466-ed22a26d85c1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck namespace: kserve-ci-e2e-test resourceVersion: "14052" uid: e3a1b01f-198d-4301-b0fe-388cdbbbdd46 kind: Event lastTimestamp: "2026-07-02T14:25:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:25:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:52Z" name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck.18be7f31410bb630 namespace: kserve-ci-e2e-test resourceVersion: "14262" uid: b68631c2-55c5-488a-ac9e-81baecfd5142 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck namespace: kserve-ci-e2e-test resourceVersion: "14052" uid: e3a1b01f-198d-4301-b0fe-388cdbbbdd46 kind: Event lastTimestamp: "2026-07-02T14:25:52Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T14:25:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:52Z" name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck.18be7f3141178464 namespace: kserve-ci-e2e-test resourceVersion: "14263" uid: 7852825b-9587-4321-993a-c713a7132074 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck namespace: kserve-ci-e2e-test resourceVersion: "14052" uid: e3a1b01f-198d-4301-b0fe-388cdbbbdd46 kind: Event lastTimestamp: "2026-07-02T14:25:54Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.713s (2.713s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T14:25:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:54Z" name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck.18be7f31e2cf2ad9 namespace: kserve-ci-e2e-test resourceVersion: "14283" uid: 3c76919f-518b-4b8f-a95c-04e7f6e3fe5a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck namespace: kserve-ci-e2e-test resourceVersion: "14052" uid: e3a1b01f-198d-4301-b0fe-388cdbbbdd46 kind: Event lastTimestamp: "2026-07-02T14:25:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:25:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:54Z" name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck.18be7f31e7235522 namespace: kserve-ci-e2e-test resourceVersion: "14285" uid: 405133d0-b5ce-4400-a399-59c3e4e59f31 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck namespace: kserve-ci-e2e-test resourceVersion: "14052" uid: e3a1b01f-198d-4301-b0fe-388cdbbbdd46 kind: Event lastTimestamp: "2026-07-02T14:25:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:25:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:54Z" name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck.18be7f31e7a15c0f namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: a7a67cd2-768f-43d1-b22a-c6c1f9fff427 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T14:25:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck namespace: kserve-ci-e2e-test resourceVersion: "14052" uid: e3a1b01f-198d-4301-b0fe-388cdbbbdd46 kind: Event lastTimestamp: "2026-07-02T14:27:10Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:25:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:10Z" name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck.18be7f31ef52021e namespace: kserve-ci-e2e-test resourceVersion: "15141" uid: 323ba49e-3d07-4cfc-9a34-6eef96950df3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck namespace: kserve-ci-e2e-test resourceVersion: "14052" uid: e3a1b01f-198d-4301-b0fe-388cdbbbdd46 kind: Event lastTimestamp: "2026-07-02T14:27:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:09Z" name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck.18be7f4351d05fd6 namespace: kserve-ci-e2e-test resourceVersion: "15096" uid: 7a7345dd-4c6d-414e-837c-13e7b1b067c3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck namespace: kserve-ci-e2e-test resourceVersion: "14052" uid: e3a1b01f-198d-4301-b0fe-388cdbbbdd46 kind: Event lastTimestamp: "2026-07-02T14:27:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:09Z" name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck.18be7f4351d116e6 namespace: kserve-ci-e2e-test resourceVersion: "15097" uid: cb5ccb9a-787c-43fb-aead-ae53d2151922 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck namespace: kserve-ci-e2e-test resourceVersion: "14052" uid: e3a1b01f-198d-4301-b0fe-388cdbbbdd46 kind: Event lastTimestamp: "2026-07-02T14:27:10Z" message: 'Readiness probe failed: Get "https://10.133.0.23:8643/healthz": dial tcp 10.133.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:10Z" name: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck.18be7f43a172553b namespace: kserve-ci-e2e-test resourceVersion: "15139" uid: dfab21c3-f4cd-42c8-aab4-7a9937cdc09a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-df2c1-predictor-688b4ccf47 namespace: kserve-ci-e2e-test resourceVersion: "14044" uid: 2075aa63-320c-45a9-b8af-992557a9da28 kind: Event lastTimestamp: "2026-07-02T14:25:35Z" message: 'Created pod: error-404-isvc-df2c1-predictor-688b4ccf47-fsdck' metadata: creationTimestamp: "2026-07-02T14:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:25:35Z" name: error-404-isvc-df2c1-predictor-688b4ccf47.18be7f2d7afb017f namespace: kserve-ci-e2e-test resourceVersion: "14051" uid: 41d1e2e4-8822-405f-bdee-e401efb7559b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-df2c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14041" uid: cacb29d5-2d08-47ed-aa82-7c0eabc4dc39 kind: Event lastTimestamp: "2026-07-02T14:25:35Z" message: Scaled up replica set error-404-isvc-df2c1-predictor-688b4ccf47 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:25:35Z" name: error-404-isvc-df2c1-predictor.18be7f2d791a5c1a namespace: kserve-ci-e2e-test resourceVersion: "14045" uid: 6fbf0496-05c5-41fc-85c3-15166efeb505 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:25:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-df2c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14043" uid: d4ca8223-d58f-4ecd-9fbb-89629ea818f8 kind: Event lastTimestamp: "2026-07-02T14:26:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:25:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:26:05Z" name: error-404-isvc-df2c1-predictor.18be7f30f7a284eb namespace: kserve-ci-e2e-test resourceVersion: "14390" uid: ec5b6d05-f98f-4b35-8153-f2a38366f989 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:25:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-df2c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14043" uid: d4ca8223-d58f-4ecd-9fbb-89629ea818f8 kind: Event lastTimestamp: "2026-07-02T14:26:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:25:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:26:05Z" name: error-404-isvc-df2c1-predictor.18be7f30f81aa1b8 namespace: kserve-ci-e2e-test resourceVersion: "14391" uid: bce2f182-0a7e-47de-a8a8-028d1d7ed0a8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:26:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-df2c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14246" uid: d4ca8223-d58f-4ecd-9fbb-89629ea818f8 kind: Event lastTimestamp: "2026-07-02T14:27:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:26:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:05Z" name: error-404-isvc-df2c1-predictor.18be7f37f58dc9b2 namespace: kserve-ci-e2e-test resourceVersion: "14958" uid: 708f1cff-34c8-4d0a-9d02-64e05214467a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:26:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-df2c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14246" uid: d4ca8223-d58f-4ecd-9fbb-89629ea818f8 kind: Event lastTimestamp: "2026-07-02T14:27:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:26:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:05Z" name: error-404-isvc-df2c1-predictor.18be7f37f5f400d0 namespace: kserve-ci-e2e-test resourceVersion: "14959" uid: c96ef69d-cd18-4b14-a6e5-7dc9aa316497 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:25:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-df2c1 namespace: kserve-ci-e2e-test resourceVersion: "14027" uid: 7f25feab-0e6b-4ba0-b349-5695d7d60261 kind: Event lastTimestamp: "2026-07-02T14:26:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:26:41Z" name: error-404-isvc-df2c1.18be7f2d7927a7f4 namespace: kserve-ci-e2e-test resourceVersion: "14717" uid: 8f2e5b39-cbeb-4f49-aaac-597aebb16c7d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-df2c1 namespace: kserve-ci-e2e-test resourceVersion: "14711" uid: 7f25feab-0e6b-4ba0-b349-5695d7d60261 kind: Event lastTimestamp: "2026-07-02T14:26:41Z" message: InferenceService [error-404-isvc-df2c1] is Ready metadata: creationTimestamp: "2026-07-02T14:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:26:41Z" name: error-404-isvc-df2c1.18be7f3cab48fcb2 namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: 56abe98a-a300-4d16-b663-0fa4ca354547 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:36:24.563059Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m namespace: kserve-ci-e2e-test resourceVersion: "19472" uid: ae07db8f-0d68-470b-afaa-b5cee9315f7e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m to ip-10-0-134-54.ec2.internal metadata: creationTimestamp: "2026-07-02T14:36:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:36:24Z" name: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m.18be7fc48857ce9e namespace: kserve-ci-e2e-test resourceVersion: "19476" uid: 964e4bbb-8c9c-411d-813f-86cb61e3bbfe reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:24Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m namespace: kserve-ci-e2e-test resourceVersion: "19474" uid: ae07db8f-0d68-470b-afaa-b5cee9315f7e kind: Event lastTimestamp: "2026-07-02T14:36:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-fbc8b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:24Z" name: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m.18be7fc4959647d5 namespace: kserve-ci-e2e-test resourceVersion: "19506" uid: f661e433-eac8-4f17-a816-deee9b9cd77f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:25Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m namespace: kserve-ci-e2e-test resourceVersion: "19481" uid: ae07db8f-0d68-470b-afaa-b5cee9315f7e kind: Event lastTimestamp: "2026-07-02T14:36:25Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:36:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:36:25Z" name: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m.18be7fc4c5f8c320 namespace: kserve-ci-e2e-test resourceVersion: "19528" uid: 3111eae9-fe64-45a1-8079-6a2d9c5c6702 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m namespace: kserve-ci-e2e-test resourceVersion: "19474" uid: ae07db8f-0d68-470b-afaa-b5cee9315f7e kind: Event lastTimestamp: "2026-07-02T14:36:25Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:36:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:25Z" name: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m.18be7fc4c77c2dba namespace: kserve-ci-e2e-test resourceVersion: "19530" uid: c05974de-180d-44e8-88a9-55b952424838 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m namespace: kserve-ci-e2e-test resourceVersion: "19474" uid: ae07db8f-0d68-470b-afaa-b5cee9315f7e kind: Event lastTimestamp: "2026-07-02T14:36:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:36:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:25Z" name: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m.18be7fc4ca605532 namespace: kserve-ci-e2e-test resourceVersion: "19531" uid: ca70e081-dc1c-401c-9526-37f89ded2014 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m namespace: kserve-ci-e2e-test resourceVersion: "19474" uid: ae07db8f-0d68-470b-afaa-b5cee9315f7e kind: Event lastTimestamp: "2026-07-02T14:36:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:36:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:25Z" name: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m.18be7fc4cad400cb namespace: kserve-ci-e2e-test resourceVersion: "19532" uid: e968fb68-be95-4628-b265-8a355f3227a5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m namespace: kserve-ci-e2e-test resourceVersion: "19474" uid: ae07db8f-0d68-470b-afaa-b5cee9315f7e kind: Event lastTimestamp: "2026-07-02T14:36:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:36:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:25Z" name: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m.18be7fc4caddae11 namespace: kserve-ci-e2e-test resourceVersion: "19533" uid: 2ea69741-6fb6-4baa-a7bc-b6ce61688863 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m namespace: kserve-ci-e2e-test resourceVersion: "19474" uid: ae07db8f-0d68-470b-afaa-b5cee9315f7e kind: Event lastTimestamp: "2026-07-02T14:36:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:36:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:25Z" name: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m.18be7fc4cdca56c5 namespace: kserve-ci-e2e-test resourceVersion: "19534" uid: f23c68ed-fdde-4e38-a4f0-9a47e5b9a685 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m namespace: kserve-ci-e2e-test resourceVersion: "19474" uid: ae07db8f-0d68-470b-afaa-b5cee9315f7e kind: Event lastTimestamp: "2026-07-02T14:36:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:36:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:25Z" name: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m.18be7fc4ce5a1162 namespace: kserve-ci-e2e-test resourceVersion: "19535" uid: 192c4fe6-b8eb-4e13-9579-c5943bc0b080 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m namespace: kserve-ci-e2e-test resourceVersion: "19474" uid: ae07db8f-0d68-470b-afaa-b5cee9315f7e kind: Event lastTimestamp: "2026-07-02T14:37:02Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:02Z" name: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m.18be7fc4f6338759 namespace: kserve-ci-e2e-test resourceVersion: "19833" uid: 88f186db-9cbc-4b9d-8694-7a6b3be96384 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m namespace: kserve-ci-e2e-test resourceVersion: "19474" uid: ae07db8f-0d68-470b-afaa-b5cee9315f7e kind: Event lastTimestamp: "2026-07-02T14:37:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:34Z" name: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m.18be7fd4e6b9cfe1 namespace: kserve-ci-e2e-test resourceVersion: "20377" uid: 70e9154d-1973-46fd-b9a2-4b5f90b80fcf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m namespace: kserve-ci-e2e-test resourceVersion: "19474" uid: ae07db8f-0d68-470b-afaa-b5cee9315f7e kind: Event lastTimestamp: "2026-07-02T14:37:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:34Z" name: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m.18be7fd4e6ba6d0c namespace: kserve-ci-e2e-test resourceVersion: "20379" uid: 8e9a3bcc-4f12-420e-b97b-8eb32df83f4a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m namespace: kserve-ci-e2e-test resourceVersion: "19474" uid: ae07db8f-0d68-470b-afaa-b5cee9315f7e kind: Event lastTimestamp: "2026-07-02T14:37:37Z" message: 'Readiness probe failed: Get "https://10.133.0.29:8643/healthz": dial tcp 10.133.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:37:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:37Z" name: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m.18be7fd57e694a1a namespace: kserve-ci-e2e-test resourceVersion: "20436" uid: 38a5b9dc-0d2f-4b65-a567-0dbaca0c08fc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-fbc8b-predictor-64c779dbb5 namespace: kserve-ci-e2e-test resourceVersion: "19464" uid: 10aa975b-7fe2-4b9c-8f88-272487659877 kind: Event lastTimestamp: "2026-07-02T14:36:24Z" message: 'Created pod: error-404-isvc-fbc8b-predictor-64c779dbb5-7br8m' metadata: creationTimestamp: "2026-07-02T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:36:24Z" name: error-404-isvc-fbc8b-predictor-64c779dbb5.18be7fc487c61aa2 namespace: kserve-ci-e2e-test resourceVersion: "19473" uid: 31237844-1d80-4b5b-a32d-565ee5a31842 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-fbc8b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19462" uid: 78f4af20-315f-45b4-a503-ceb602c2c8f3 kind: Event lastTimestamp: "2026-07-02T14:36:24Z" message: Scaled up replica set error-404-isvc-fbc8b-predictor-64c779dbb5 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:36:24Z" name: error-404-isvc-fbc8b-predictor.18be7fc485fe9605 namespace: kserve-ci-e2e-test resourceVersion: "19465" uid: f36dd3a0-06c6-49fc-85a3-289af19769b7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fbc8b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19463" uid: 94d29d45-0806-488d-a891-4eee3eb59750 kind: Event lastTimestamp: "2026-07-02T14:36:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:36:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:36:39Z" name: error-404-isvc-fbc8b-predictor.18be7fc80477274e namespace: kserve-ci-e2e-test resourceVersion: "19656" uid: 5134aad6-af38-457e-8dfb-57b76f84df7f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fbc8b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19463" uid: 94d29d45-0806-488d-a891-4eee3eb59750 kind: Event lastTimestamp: "2026-07-02T14:36:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:36:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:36:39Z" name: error-404-isvc-fbc8b-predictor.18be7fc804db9739 namespace: kserve-ci-e2e-test resourceVersion: "19658" uid: 6c46f9cc-df1e-4f48-a0a9-60ef8cf0fc20 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:36:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fbc8b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19657" uid: 94d29d45-0806-488d-a891-4eee3eb59750 kind: Event lastTimestamp: "2026-07-02T14:37:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:36:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:37:24Z" name: error-404-isvc-fbc8b-predictor.18be7fcb838e0952 namespace: kserve-ci-e2e-test resourceVersion: "20176" uid: 24eea28b-b747-48ee-92d4-8dafdddaeeee reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:36:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fbc8b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19657" uid: 94d29d45-0806-488d-a891-4eee3eb59750 kind: Event lastTimestamp: "2026-07-02T14:37:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:36:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:37:24Z" name: error-404-isvc-fbc8b-predictor.18be7fcb83fdd57f namespace: kserve-ci-e2e-test resourceVersion: "20177" uid: 9628461c-3b07-4c14-9dd8-c45dea0da319 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:36:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fbc8b namespace: kserve-ci-e2e-test resourceVersion: "19445" uid: 4abbffe5-5fdd-4f5e-b0b8-c0cc61c63b32 kind: Event lastTimestamp: "2026-07-02T14:37:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:37:12Z" name: error-404-isvc-fbc8b.18be7fc486094dce namespace: kserve-ci-e2e-test resourceVersion: "19921" uid: cb696fd2-34c8-4670-afac-4a9bb394de3b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fbc8b namespace: kserve-ci-e2e-test resourceVersion: "19445" uid: 4abbffe5-5fdd-4f5e-b0b8-c0cc61c63b32 kind: Event lastTimestamp: "2026-07-02T14:36:24Z" message: 'Failed to update status for InferenceService "error-404-isvc-fbc8b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fbc8b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:36:24Z" name: error-404-isvc-fbc8b.18be7fc48691bb68 namespace: kserve-ci-e2e-test resourceVersion: "19468" uid: 936ae57b-7004-4849-8e6e-31746ebba630 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fbc8b namespace: kserve-ci-e2e-test resourceVersion: "19445" uid: 4abbffe5-5fdd-4f5e-b0b8-c0cc61c63b32 kind: Event lastTimestamp: "2026-07-02T14:36:24Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fbc8b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:36:24Z" name: error-404-isvc-fbc8b.18be7fc486922984 namespace: kserve-ci-e2e-test resourceVersion: "19470" uid: b8b9bfa7-1313-462d-9b9a-7b96977fdf2e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fbc8b namespace: kserve-ci-e2e-test resourceVersion: "19919" uid: 4abbffe5-5fdd-4f5e-b0b8-c0cc61c63b32 kind: Event lastTimestamp: "2026-07-02T14:37:12Z" message: InferenceService [error-404-isvc-fbc8b] is Ready metadata: creationTimestamp: "2026-07-02T14:37:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:37:12Z" name: error-404-isvc-fbc8b.18be7fcfb2065335 namespace: kserve-ci-e2e-test resourceVersion: "19920" uid: 31207cc4-34ad-4671-aeda-9ac4b541672a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:25:35.227046Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd namespace: kserve-ci-e2e-test resourceVersion: "13943" uid: c4c5f264-e7f2-42b8-8791-2f83394a4322 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd to ip-10-0-143-132.ec2.internal metadata: creationTimestamp: "2026-07-02T14:25:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:25:35Z" name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd.18be7f2d58e67a0e namespace: kserve-ci-e2e-test resourceVersion: "13950" uid: 6c38eee0-17f7-4b44-bfb0-8ff4ea64ab77 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:35Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: c4c5f264-e7f2-42b8-8791-2f83394a4322 kind: Event lastTimestamp: "2026-07-02T14:25:35Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:35Z" name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd.18be7f2d6651bdfe namespace: kserve-ci-e2e-test resourceVersion: "13990" uid: ea2cc4dd-e79b-484b-a525-b44c1f3f73f8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:36Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd namespace: kserve-ci-e2e-test resourceVersion: "13955" uid: c4c5f264-e7f2-42b8-8791-2f83394a4322 kind: Event lastTimestamp: "2026-07-02T14:25:36Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:25:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:25:36Z" name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd.18be7f2d964c76b0 namespace: kserve-ci-e2e-test resourceVersion: "14095" uid: 507bdf67-92dd-4887-b9c1-38042b3a125a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: c4c5f264-e7f2-42b8-8791-2f83394a4322 kind: Event lastTimestamp: "2026-07-02T14:25:36Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8e2b89e7009febc315fad3ff8d06e5aec68b35b7cc22df79daf7c1334f909f5a" already present on machine metadata: creationTimestamp: "2026-07-02T14:25:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:36Z" name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd.18be7f2d97df05cd namespace: kserve-ci-e2e-test resourceVersion: "14099" uid: f2f204ad-7f00-4d37-9d7c-f7ed97ab4bbc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: c4c5f264-e7f2-42b8-8791-2f83394a4322 kind: Event lastTimestamp: "2026-07-02T14:25:36Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:25:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:36Z" name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd.18be7f2d9b70d19c namespace: kserve-ci-e2e-test resourceVersion: "14102" uid: 630fe3dc-8153-42ff-ac5a-7883c8b130bc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: c4c5f264-e7f2-42b8-8791-2f83394a4322 kind: Event lastTimestamp: "2026-07-02T14:25:36Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:25:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:36Z" name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd.18be7f2d9be1ad33 namespace: kserve-ci-e2e-test resourceVersion: "14104" uid: e914cda8-50d7-4112-b5d3-e0ceb05a011f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: c4c5f264-e7f2-42b8-8791-2f83394a4322 kind: Event lastTimestamp: "2026-07-02T14:25:40Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1697" metadata: creationTimestamp: "2026-07-02T14:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:40Z" name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd.18be7f2e846a9dd2 namespace: kserve-ci-e2e-test resourceVersion: "14147" uid: 1f6edd88-12b4-4d5e-b1db-26257e1d0fe8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: c4c5f264-e7f2-42b8-8791-2f83394a4322 kind: Event lastTimestamp: "2026-07-02T14:26:07Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1697" in 27.333s (27.333s including waiting). Image size: 1555893189 bytes.' metadata: creationTimestamp: "2026-07-02T14:26:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:07Z" name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd.18be7f34e1a0d521 namespace: kserve-ci-e2e-test resourceVersion: "14405" uid: 738aec93-482c-46ae-8611-e57946e84c66 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: c4c5f264-e7f2-42b8-8791-2f83394a4322 kind: Event lastTimestamp: "2026-07-02T14:26:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:26:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:07Z" name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd.18be7f34e799372e namespace: kserve-ci-e2e-test resourceVersion: "14408" uid: 5c7650b9-2606-4c1e-bea4-0ac3ea1aa2b0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: c4c5f264-e7f2-42b8-8791-2f83394a4322 kind: Event lastTimestamp: "2026-07-02T14:26:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:26:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:07Z" name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd.18be7f34e88e9523 namespace: kserve-ci-e2e-test resourceVersion: "14410" uid: f694e6dd-f275-4b01-af2f-6af511b85f87 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: c4c5f264-e7f2-42b8-8791-2f83394a4322 kind: Event lastTimestamp: "2026-07-02T14:26:07Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T14:26:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:07Z" name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd.18be7f34e89c7b27 namespace: kserve-ci-e2e-test resourceVersion: "14412" uid: 156f1b1f-6367-423f-b632-ef88432b6216 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: c4c5f264-e7f2-42b8-8791-2f83394a4322 kind: Event lastTimestamp: "2026-07-02T14:26:10Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.847s (2.847s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T14:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:10Z" name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd.18be7f359258a36d namespace: kserve-ci-e2e-test resourceVersion: "14435" uid: 1cd0319e-e19b-4b66-8c7f-fdd000169bfa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: c4c5f264-e7f2-42b8-8791-2f83394a4322 kind: Event lastTimestamp: "2026-07-02T14:26:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:10Z" name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd.18be7f3596afe9a5 namespace: kserve-ci-e2e-test resourceVersion: "14439" uid: 5927ef17-1d7c-4c04-b919-cb2bca6d53d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: c4c5f264-e7f2-42b8-8791-2f83394a4322 kind: Event lastTimestamp: "2026-07-02T14:26:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:10Z" name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd.18be7f359714e778 namespace: kserve-ci-e2e-test resourceVersion: "14442" uid: ce62c3f2-b4d6-41d5-947d-cf06b5ab95be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T14:26:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: c4c5f264-e7f2-42b8-8791-2f83394a4322 kind: Event lastTimestamp: "2026-07-02T14:28:07Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:07Z" name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd.18be7f35cb71f178 namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: a913ddf7-3f8b-49d1-8238-0c70ca727077 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: c4c5f264-e7f2-42b8-8791-2f83394a4322 kind: Event lastTimestamp: "2026-07-02T14:28:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:28:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:05Z" name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd.18be7f5052979f66 namespace: kserve-ci-e2e-test resourceVersion: "15749" uid: 66d7b8de-9e88-46cb-b74a-3e17d23570dd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: c4c5f264-e7f2-42b8-8791-2f83394a4322 kind: Event lastTimestamp: "2026-07-02T14:28:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:28:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:05Z" name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd.18be7f50529828be namespace: kserve-ci-e2e-test resourceVersion: "15752" uid: 7c95adb0-c63c-4d08-8b05-275a10a2d00d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd namespace: kserve-ci-e2e-test resourceVersion: "13947" uid: c4c5f264-e7f2-42b8-8791-2f83394a4322 kind: Event lastTimestamp: "2026-07-02T14:28:07Z" message: 'Readiness probe failed: Get "https://10.132.0.25:8643/healthz": dial tcp 10.132.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:28:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:07Z" name: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd.18be7f50cdd5e5f8 namespace: kserve-ci-e2e-test resourceVersion: "15832" uid: bc0c7775-19b6-4eee-a5e4-e385be276123 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-5b4cb68696 namespace: kserve-ci-e2e-test resourceVersion: "13930" uid: 953977a7-224b-4474-96a9-856257d2cdee kind: Event lastTimestamp: "2026-07-02T14:25:35Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-5b4cb68696-qzgmd' metadata: creationTimestamp: "2026-07-02T14:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:25:35Z" name: isvc-sklearn-graph-1-predictor-5b4cb68696.18be7f2d58252686 namespace: kserve-ci-e2e-test resourceVersion: "13945" uid: d80b3b9e-fb77-4682-9313-c28fdfd187e1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13929" uid: 5b98c3e7-4445-4fa7-a860-fa296b4e83d4 kind: Event lastTimestamp: "2026-07-02T14:25:35Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-5b4cb68696 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:25:35Z" name: isvc-sklearn-graph-1-predictor.18be7f2d4f8e01b0 namespace: kserve-ci-e2e-test resourceVersion: "13931" uid: 62ed7aea-99bc-44fb-8e92-b17d1f05e1e1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:25:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13937" uid: 6275ce67-173f-40e9-a7a9-73f8c2720472 kind: Event lastTimestamp: "2026-07-02T14:26:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:25:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:26:35Z" name: isvc-sklearn-graph-1-predictor.18be7f30d519eaef namespace: kserve-ci-e2e-test resourceVersion: "14643" uid: 2731d172-1314-4441-a096-e494fc080e47 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:25:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13937" uid: 6275ce67-173f-40e9-a7a9-73f8c2720472 kind: Event lastTimestamp: "2026-07-02T14:26:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:25:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:26:35Z" name: isvc-sklearn-graph-1-predictor.18be7f30d58c6e27 namespace: kserve-ci-e2e-test resourceVersion: "14644" uid: 6c9e3d0e-f10e-4a4a-896a-068856433d84 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:26:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: 6275ce67-173f-40e9-a7a9-73f8c2720472 kind: Event lastTimestamp: "2026-07-02T14:27:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:26:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:50Z" name: isvc-sklearn-graph-1-predictor.18be7f3ed1f1ea6f namespace: kserve-ci-e2e-test resourceVersion: "15486" uid: ca929c71-921b-4ea8-9498-46b86f9409f4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:26:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: 6275ce67-173f-40e9-a7a9-73f8c2720472 kind: Event lastTimestamp: "2026-07-02T14:27:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:26:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:50Z" name: isvc-sklearn-graph-1-predictor.18be7f3ed25c6798 namespace: kserve-ci-e2e-test resourceVersion: "15487" uid: cfb918fd-d6b9-4fa6-9160-4bd2572dcaf5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T14:25:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13919" uid: f30404f9-883e-46f8-be24-30d30445491d kind: Event lastTimestamp: "2026-07-02T14:27:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:27:27Z" name: isvc-sklearn-graph-1.18be7f2d55d7de58 namespace: kserve-ci-e2e-test resourceVersion: "15303" uid: 115a05bf-b833-4806-a79f-5a55fc64f570 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15300" uid: f30404f9-883e-46f8-be24-30d30445491d kind: Event lastTimestamp: "2026-07-02T14:27:27Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-02T14:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:27:27Z" name: isvc-sklearn-graph-1.18be7f4783ac39ef namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: 9a876daf-ede7-40dd-9e97-e002251433dc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:25:35.722883Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4 namespace: kserve-ci-e2e-test resourceVersion: "14028" uid: f347f490-0755-4416-bec9-0877c3b74d06 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4 to ip-10-0-143-132.ec2.internal metadata: creationTimestamp: "2026-07-02T14:25:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:25:35Z" name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4.18be7f2d767461a2 namespace: kserve-ci-e2e-test resourceVersion: "14032" uid: cb515d8d-dd38-4dab-96bb-86b01897f938 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:35Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4 namespace: kserve-ci-e2e-test resourceVersion: "14031" uid: f347f490-0755-4416-bec9-0877c3b74d06 kind: Event lastTimestamp: "2026-07-02T14:25:35Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:35Z" name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4.18be7f2d847dd205 namespace: kserve-ci-e2e-test resourceVersion: "14071" uid: f6f085ab-f251-44d9-90f8-226bf3499c10 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:36Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4 namespace: kserve-ci-e2e-test resourceVersion: "14042" uid: f347f490-0755-4416-bec9-0877c3b74d06 kind: Event lastTimestamp: "2026-07-02T14:25:36Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:25:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:25:36Z" name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4.18be7f2db3d32dcd namespace: kserve-ci-e2e-test resourceVersion: "14108" uid: 2526c02e-9ac3-4e48-a88f-cfb006e6475c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4 namespace: kserve-ci-e2e-test resourceVersion: "14031" uid: f347f490-0755-4416-bec9-0877c3b74d06 kind: Event lastTimestamp: "2026-07-02T14:25:36Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8e2b89e7009febc315fad3ff8d06e5aec68b35b7cc22df79daf7c1334f909f5a" already present on machine metadata: creationTimestamp: "2026-07-02T14:25:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:36Z" name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4.18be7f2db55b55a7 namespace: kserve-ci-e2e-test resourceVersion: "14110" uid: 8544a345-6ba1-4857-8720-f298c14ec9bb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4 namespace: kserve-ci-e2e-test resourceVersion: "14031" uid: f347f490-0755-4416-bec9-0877c3b74d06 kind: Event lastTimestamp: "2026-07-02T14:25:36Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:25:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:36Z" name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4.18be7f2db80ff3ac namespace: kserve-ci-e2e-test resourceVersion: "14112" uid: 3d103d4f-490a-4737-9cb5-0510c44be929 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4 namespace: kserve-ci-e2e-test resourceVersion: "14031" uid: f347f490-0755-4416-bec9-0877c3b74d06 kind: Event lastTimestamp: "2026-07-02T14:25:36Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:25:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:36Z" name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4.18be7f2db8809cf6 namespace: kserve-ci-e2e-test resourceVersion: "14113" uid: 10ea660c-f646-44ca-86ee-2c4ed42d4aa9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4 namespace: kserve-ci-e2e-test resourceVersion: "14031" uid: f347f490-0755-4416-bec9-0877c3b74d06 kind: Event lastTimestamp: "2026-07-02T14:25:41Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1697" metadata: creationTimestamp: "2026-07-02T14:25:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:41Z" name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4.18be7f2ec140d330 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: b6ff3fd2-af11-470f-bfca-d0ffecacd899 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4 namespace: kserve-ci-e2e-test resourceVersion: "14031" uid: f347f490-0755-4416-bec9-0877c3b74d06 kind: Event lastTimestamp: "2026-07-02T14:26:07Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1697" in 26.312s (26.312s including waiting). Image size: 1555893189 bytes.' metadata: creationTimestamp: "2026-07-02T14:26:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:07Z" name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4.18be7f34e1a0d6b2 namespace: kserve-ci-e2e-test resourceVersion: "14406" uid: 8d8b862e-0e2d-4887-9ca9-d6d4a8fc7e17 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4 namespace: kserve-ci-e2e-test resourceVersion: "14031" uid: f347f490-0755-4416-bec9-0877c3b74d06 kind: Event lastTimestamp: "2026-07-02T14:26:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:26:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:07Z" name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4.18be7f34e75a4791 namespace: kserve-ci-e2e-test resourceVersion: "14407" uid: 70f2b59b-5a49-45a2-976b-e2d0d73b155b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4 namespace: kserve-ci-e2e-test resourceVersion: "14031" uid: f347f490-0755-4416-bec9-0877c3b74d06 kind: Event lastTimestamp: "2026-07-02T14:26:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:26:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:07Z" name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4.18be7f34e88e98b2 namespace: kserve-ci-e2e-test resourceVersion: "14409" uid: c105b57a-225e-4a47-8a36-48435ec6d304 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4 namespace: kserve-ci-e2e-test resourceVersion: "14031" uid: f347f490-0755-4416-bec9-0877c3b74d06 kind: Event lastTimestamp: "2026-07-02T14:26:07Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T14:26:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:07Z" name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4.18be7f34e89b9e33 namespace: kserve-ci-e2e-test resourceVersion: "14411" uid: b7fc91fe-30b8-4b00-bc5f-92b205ea2a43 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4 namespace: kserve-ci-e2e-test resourceVersion: "14031" uid: f347f490-0755-4416-bec9-0877c3b74d06 kind: Event lastTimestamp: "2026-07-02T14:26:10Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.849s (2.849s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T14:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:10Z" name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4.18be7f35927b2349 namespace: kserve-ci-e2e-test resourceVersion: "14436" uid: afd06079-e83f-436f-abc8-34f41f3533b3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4 namespace: kserve-ci-e2e-test resourceVersion: "14031" uid: f347f490-0755-4416-bec9-0877c3b74d06 kind: Event lastTimestamp: "2026-07-02T14:26:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:10Z" name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4.18be7f3596ad85e3 namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: 796bc809-93c0-4737-941e-0a1d6953251d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4 namespace: kserve-ci-e2e-test resourceVersion: "14031" uid: f347f490-0755-4416-bec9-0877c3b74d06 kind: Event lastTimestamp: "2026-07-02T14:26:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:10Z" name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4.18be7f359710f713 namespace: kserve-ci-e2e-test resourceVersion: "14441" uid: 430f8fa0-f8b7-4a48-a6de-6d100185c137 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T14:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4 namespace: kserve-ci-e2e-test resourceVersion: "14031" uid: f347f490-0755-4416-bec9-0877c3b74d06 kind: Event lastTimestamp: "2026-07-02T14:28:08Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:08Z" name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4.18be7f360750bc90 namespace: kserve-ci-e2e-test resourceVersion: "15845" uid: 758d761c-ee64-4aeb-a5ac-1d6717b7295a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4 namespace: kserve-ci-e2e-test resourceVersion: "14031" uid: f347f490-0755-4416-bec9-0877c3b74d06 kind: Event lastTimestamp: "2026-07-02T14:28:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:28:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:05Z" name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4.18be7f504e1172e5 namespace: kserve-ci-e2e-test resourceVersion: "15722" uid: f4241a87-e74b-42c6-a401-eed054a6b0b0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4 namespace: kserve-ci-e2e-test resourceVersion: "14031" uid: f347f490-0755-4416-bec9-0877c3b74d06 kind: Event lastTimestamp: "2026-07-02T14:28:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:28:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:05Z" name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4.18be7f504e121a6c namespace: kserve-ci-e2e-test resourceVersion: "15724" uid: 6595be56-0303-4b1d-9858-99adae7e5377 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4 namespace: kserve-ci-e2e-test resourceVersion: "14031" uid: f347f490-0755-4416-bec9-0877c3b74d06 kind: Event lastTimestamp: "2026-07-02T14:28:08Z" message: 'Readiness probe failed: Get "https://10.132.0.27:8643/healthz": dial tcp 10.132.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:08Z" name: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4.18be7f5109b70c09 namespace: kserve-ci-e2e-test resourceVersion: "15844" uid: b0ec71d9-eea7-4fd5-a9fe-3b6d57b2f017 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-5fc64bd448 namespace: kserve-ci-e2e-test resourceVersion: "14020" uid: 54d031ee-5cbf-4378-a838-02b5238b0733 kind: Event lastTimestamp: "2026-07-02T14:25:35Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-5fc64bd448-nh4b4' metadata: creationTimestamp: "2026-07-02T14:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:25:35Z" name: isvc-sklearn-graph-2-predictor-5fc64bd448.18be7f2d75b38f6e namespace: kserve-ci-e2e-test resourceVersion: "14029" uid: 8d93eece-88f3-4ff6-93cd-de595445bb9d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14019" uid: bcee5d68-1820-453c-b838-b85ccb7f92dd kind: Event lastTimestamp: "2026-07-02T14:25:35Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-5fc64bd448 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:25:35Z" name: isvc-sklearn-graph-2-predictor.18be7f2d6f05e1e9 namespace: kserve-ci-e2e-test resourceVersion: "14022" uid: 3fc1356b-db99-4d9b-8831-797f01a42665 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:25:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: afc4c7e9-8751-45bc-9efd-9b8b406756e2 kind: Event lastTimestamp: "2026-07-02T14:26:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:25:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:26:35Z" name: isvc-sklearn-graph-2-predictor.18be7f30ede8b134 namespace: kserve-ci-e2e-test resourceVersion: "14654" uid: 032c652f-0f01-4c3e-823e-952aa5e54b2c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:25:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: afc4c7e9-8751-45bc-9efd-9b8b406756e2 kind: Event lastTimestamp: "2026-07-02T14:26:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:25:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:26:35Z" name: isvc-sklearn-graph-2-predictor.18be7f30ee4c9b7b namespace: kserve-ci-e2e-test resourceVersion: "14655" uid: a02ce66a-53d2-4a46-a929-7f651b45e19f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:26:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14240" uid: afc4c7e9-8751-45bc-9efd-9b8b406756e2 kind: Event lastTimestamp: "2026-07-02T14:27:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:26:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:50Z" name: isvc-sklearn-graph-2-predictor.18be7f3ee96e8e57 namespace: kserve-ci-e2e-test resourceVersion: "15494" uid: 6d60ad50-8ee1-4d6e-b917-d5f1eddca419 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:26:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14240" uid: afc4c7e9-8751-45bc-9efd-9b8b406756e2 kind: Event lastTimestamp: "2026-07-02T14:27:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:26:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:50Z" name: isvc-sklearn-graph-2-predictor.18be7f3ee9eef766 namespace: kserve-ci-e2e-test resourceVersion: "15495" uid: 3db1bc3e-a823-49a8-973c-d4a5944cc61a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T14:25:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14000" uid: 49f79b26-4caf-496b-becb-e8e15d8aee1e kind: Event lastTimestamp: "2026-07-02T14:27:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:27:28Z" name: isvc-sklearn-graph-2.18be7f2d6fd4c068 namespace: kserve-ci-e2e-test resourceVersion: "15319" uid: 4d64fe65-6dea-4186-a98b-42aafdd43c88 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15317" uid: 49f79b26-4caf-496b-becb-e8e15d8aee1e kind: Event lastTimestamp: "2026-07-02T14:27:28Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-02T14:27:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:27:28Z" name: isvc-sklearn-graph-2.18be7f47bf9c1005 namespace: kserve-ci-e2e-test resourceVersion: "15318" uid: 91ffc1ae-1a37-4de8-aff0-c07f903e0537 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:25:35.537889Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5 namespace: kserve-ci-e2e-test resourceVersion: "14002" uid: 5df30d29-772b-4747-8181-b2610772dfd8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-wrtz5 to ip-10-0-143-132.ec2.internal metadata: creationTimestamp: "2026-07-02T14:25:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:25:35Z" name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5.18be7f2d6b6d9983 namespace: kserve-ci-e2e-test resourceVersion: "14008" uid: 490ac8bf-c98e-40e0-8b17-1dbaf8d6a125 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:35Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5 namespace: kserve-ci-e2e-test resourceVersion: "14017" uid: 5df30d29-772b-4747-8181-b2610772dfd8 kind: Event lastTimestamp: "2026-07-02T14:25:35Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:25:35Z" name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5.18be7f2d84f31635 namespace: kserve-ci-e2e-test resourceVersion: "14073" uid: b24e1559-bf18-487c-aefa-2bf01836c09c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5 namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 5df30d29-772b-4747-8181-b2610772dfd8 kind: Event lastTimestamp: "2026-07-02T14:25:35Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:8e2b89e7009febc315fad3ff8d06e5aec68b35b7cc22df79daf7c1334f909f5a" already present on machine metadata: creationTimestamp: "2026-07-02T14:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:35Z" name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5.18be7f2d86628757 namespace: kserve-ci-e2e-test resourceVersion: "14075" uid: 864b0c56-e644-4afb-bc22-d11f385cbe35 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5 namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 5df30d29-772b-4747-8181-b2610772dfd8 kind: Event lastTimestamp: "2026-07-02T14:25:36Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T14:25:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:36Z" name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5.18be7f2d892b3574 namespace: kserve-ci-e2e-test resourceVersion: "14080" uid: 6e5ffa6f-5fdf-48bd-a79b-137c57f94854 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5 namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 5df30d29-772b-4747-8181-b2610772dfd8 kind: Event lastTimestamp: "2026-07-02T14:25:36Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T14:25:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:36Z" name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5.18be7f2d899aab71 namespace: kserve-ci-e2e-test resourceVersion: "14081" uid: 6803a211-d34d-4673-b8b5-c14508553e8c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5 namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 5df30d29-772b-4747-8181-b2610772dfd8 kind: Event lastTimestamp: "2026-07-02T14:25:40Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-02T14:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:40Z" name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5.18be7f2e8485488f namespace: kserve-ci-e2e-test resourceVersion: "14148" uid: 950607cd-73ab-4bcb-80b9-7644dc065fdf reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5 namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 5df30d29-772b-4747-8181-b2610772dfd8 kind: Event lastTimestamp: "2026-07-02T14:26:08Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 27.764s (27.764s including waiting). Image size: 1305293048 bytes.' metadata: creationTimestamp: "2026-07-02T14:26:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:08Z" name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5.18be7f34fb6d32ec namespace: kserve-ci-e2e-test resourceVersion: "14415" uid: ea045fbf-a784-407c-b78e-d6f3e28307db reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5 namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 5df30d29-772b-4747-8181-b2610772dfd8 kind: Event lastTimestamp: "2026-07-02T14:26:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:26:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:08Z" name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5.18be7f3500b323a2 namespace: kserve-ci-e2e-test resourceVersion: "14416" uid: 043a4293-7d11-4505-abf8-55be3d16c582 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5 namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 5df30d29-772b-4747-8181-b2610772dfd8 kind: Event lastTimestamp: "2026-07-02T14:26:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:26:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:08Z" name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5.18be7f3501c52fbe namespace: kserve-ci-e2e-test resourceVersion: "14417" uid: ca60a384-2c9e-4711-b567-5e5fc073b4a0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5 namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 5df30d29-772b-4747-8181-b2610772dfd8 kind: Event lastTimestamp: "2026-07-02T14:26:08Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T14:26:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:08Z" name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5.18be7f3501d4282a namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: 266fc157-5838-48b6-ad48-1ff53192a39a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5 namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 5df30d29-772b-4747-8181-b2610772dfd8 kind: Event lastTimestamp: "2026-07-02T14:26:10Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.433s (2.433s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T14:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:10Z" name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5.18be7f3592e3fb75 namespace: kserve-ci-e2e-test resourceVersion: "14437" uid: c5ca000c-b695-440f-bf22-1ff93879e873 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5 namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 5df30d29-772b-4747-8181-b2610772dfd8 kind: Event lastTimestamp: "2026-07-02T14:26:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:10Z" name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5.18be7f3596b7789e namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: b84f83e3-1a0a-48eb-bd1c-bff2574e29b5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5 namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 5df30d29-772b-4747-8181-b2610772dfd8 kind: Event lastTimestamp: "2026-07-02T14:26:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:10Z" name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5.18be7f35972f4f20 namespace: kserve-ci-e2e-test resourceVersion: "14443" uid: acb61dfb-c9d4-4275-b25c-38f53bdf80a9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T14:26:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5 namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 5df30d29-772b-4747-8181-b2610772dfd8 kind: Event lastTimestamp: "2026-07-02T14:28:07Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5.18be7f35cb306b3e namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: 9e22d1e8-63bf-424d-b268-de0a5e89711d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5 namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 5df30d29-772b-4747-8181-b2610772dfd8 kind: Event lastTimestamp: "2026-07-02T14:28:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:28:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:05Z" name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5.18be7f504ab56e4a namespace: kserve-ci-e2e-test resourceVersion: "15694" uid: 194864c2-0802-4516-b0cd-1b9405a409be reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5 namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 5df30d29-772b-4747-8181-b2610772dfd8 kind: Event lastTimestamp: "2026-07-02T14:28:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:28:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:05Z" name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5.18be7f504ab5dae1 namespace: kserve-ci-e2e-test resourceVersion: "15696" uid: 5d7b034c-b968-41bf-97f1-7e3f07f4148f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5 namespace: kserve-ci-e2e-test resourceVersion: "14005" uid: 5df30d29-772b-4747-8181-b2610772dfd8 kind: Event lastTimestamp: "2026-07-02T14:28:07Z" message: 'Readiness probe failed: Get "https://10.132.0.26:8643/healthz": dial tcp 10.132.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:28:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-wrtz5.18be7f50cdd7719b namespace: kserve-ci-e2e-test resourceVersion: "15833" uid: db698074-39c8-4ea8-b7a0-1af389bd5164 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-132.ec2.internal source: component: kubelet host: ip-10-0-143-132.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "13993" uid: 03daabd1-f313-4d17-a4f2-73c82d839f05 kind: Event lastTimestamp: "2026-07-02T14:25:35Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-wrtz5' metadata: creationTimestamp: "2026-07-02T14:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:25:35Z" name: isvc-xgboost-graph-predictor-555fc69d56.18be7f2d6ad03fb9 namespace: kserve-ci-e2e-test resourceVersion: "14003" uid: 899718fd-f4b0-4cad-a778-44e14c99ba11 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13989" uid: 2dfb4b94-979e-414e-b28c-1e76fef0e615 kind: Event lastTimestamp: "2026-07-02T14:25:35Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:25:35Z" name: isvc-xgboost-graph-predictor.18be7f2d67511cc7 namespace: kserve-ci-e2e-test resourceVersion: "13994" uid: 1ad82e82-6aa5-4330-8cd4-a3ccf1025204 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:25:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13992" uid: 572df150-c4f7-4f64-b7aa-155440c4c4b9 kind: Event lastTimestamp: "2026-07-02T14:26:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:25:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:26:35Z" name: isvc-xgboost-graph-predictor.18be7f30e5d4d1b1 namespace: kserve-ci-e2e-test resourceVersion: "14651" uid: 346fece1-742d-4da3-af32-4ff1a5785c50 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:25:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13992" uid: 572df150-c4f7-4f64-b7aa-155440c4c4b9 kind: Event lastTimestamp: "2026-07-02T14:26:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:25:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:26:35Z" name: isvc-xgboost-graph-predictor.18be7f30e646ad37 namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: b79f860a-25f0-4724-b25b-1457a92ffacd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:26:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14238" uid: 572df150-c4f7-4f64-b7aa-155440c4c4b9 kind: Event lastTimestamp: "2026-07-02T14:27:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:26:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:50Z" name: isvc-xgboost-graph-predictor.18be7f3ee1d70a52 namespace: kserve-ci-e2e-test resourceVersion: "15490" uid: 00274ecd-c170-4ccb-9c2f-56746ede718e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:26:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14238" uid: 572df150-c4f7-4f64-b7aa-155440c4c4b9 kind: Event lastTimestamp: "2026-07-02T14:27:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:26:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:50Z" name: isvc-xgboost-graph-predictor.18be7f3ee2720f43 namespace: kserve-ci-e2e-test resourceVersion: "15491" uid: 87982528-a6bd-400b-a756-d65a1a25df95 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T14:25:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "13980" uid: 60953858-7238-4bf5-898a-088adad83c5c kind: Event lastTimestamp: "2026-07-02T14:27:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:27:17Z" name: isvc-xgboost-graph.18be7f2d675b47a6 namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: e8e13289-ad47-4a39-a75f-a5d3f4009790 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 60953858-7238-4bf5-898a-088adad83c5c kind: Event lastTimestamp: "2026-07-02T14:27:17Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-02T14:27:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:27:17Z" name: isvc-xgboost-graph.18be7f4530bb46c8 namespace: kserve-ci-e2e-test resourceVersion: "15210" uid: 2aad8d01-6896-4f32-9bba-072575a3cccc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:27:55.102869Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7d6d5bb5c4-kqn2s namespace: kserve-ci-e2e-test resourceVersion: "15538" uid: d80103d8-904b-4db7-8756-141cc8462374 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-7d6d5bb5c4-kqn2s to ip-10-0-143-249.ec2.internal metadata: creationTimestamp: "2026-07-02T14:27:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:27:55Z" name: model-chainer-7d6d5bb5c4-kqn2s.18be7f4dea2645aa namespace: kserve-ci-e2e-test resourceVersion: "15543" uid: 0ed2bd24-5da0-4896-918e-a9799a82a9eb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:55Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7d6d5bb5c4-kqn2s namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: d80103d8-904b-4db7-8756-141cc8462374 kind: Event lastTimestamp: "2026-07-02T14:27:55Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:27:55Z" name: model-chainer-7d6d5bb5c4-kqn2s.18be7f4e03c7570c namespace: kserve-ci-e2e-test resourceVersion: "15558" uid: ec89f376-38d3-4ae6-8d13-4cc021b69527 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7d6d5bb5c4-kqn2s namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: d80103d8-904b-4db7-8756-141cc8462374 kind: Event lastTimestamp: "2026-07-02T14:27:55Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:e0bdb71677d5eb7ee4784378e591e8126d0dcafbbe991e5dcf178c67e85889cd" already present on machine metadata: creationTimestamp: "2026-07-02T14:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:55Z" name: model-chainer-7d6d5bb5c4-kqn2s.18be7f4e05598de9 namespace: kserve-ci-e2e-test resourceVersion: "15560" uid: 0034f0b7-bf3e-4dba-a242-bd568251c468 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7d6d5bb5c4-kqn2s namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: d80103d8-904b-4db7-8756-141cc8462374 kind: Event lastTimestamp: "2026-07-02T14:27:55Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-02T14:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:55Z" name: model-chainer-7d6d5bb5c4-kqn2s.18be7f4e0800c3f1 namespace: kserve-ci-e2e-test resourceVersion: "15561" uid: b02e4415-93d5-4256-a965-5e9a61874756 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7d6d5bb5c4-kqn2s namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: d80103d8-904b-4db7-8756-141cc8462374 kind: Event lastTimestamp: "2026-07-02T14:27:55Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-02T14:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:55Z" name: model-chainer-7d6d5bb5c4-kqn2s.18be7f4e086802c1 namespace: kserve-ci-e2e-test resourceVersion: "15562" uid: b5f95c9d-84c7-4758-ba74-cd1f9740b136 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7d6d5bb5c4-kqn2s namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: d80103d8-904b-4db7-8756-141cc8462374 kind: Event lastTimestamp: "2026-07-02T14:28:05Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-02T14:28:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:05Z" name: model-chainer-7d6d5bb5c4-kqn2s.18be7f504335e521 namespace: kserve-ci-e2e-test resourceVersion: "15673" uid: add97086-bc06-4fc9-9df9-fefccfb2c88e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:28:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7d6d5bb5c4-kqn2s namespace: kserve-ci-e2e-test resourceVersion: "15540" uid: d80103d8-904b-4db7-8756-141cc8462374 kind: Event lastTimestamp: "2026-07-02T14:28:32Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:28:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:32Z" name: model-chainer-7d6d5bb5c4-kqn2s.18be7f50b2413308 namespace: kserve-ci-e2e-test resourceVersion: "16088" uid: bfff146a-69fd-4d3a-bbfb-ec15f8eb4252 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-7d6d5bb5c4 namespace: kserve-ci-e2e-test resourceVersion: "15534" uid: b152f20f-885c-44c7-a8b5-681bd317631c kind: Event lastTimestamp: "2026-07-02T14:27:55Z" message: 'Created pod: model-chainer-7d6d5bb5c4-kqn2s' metadata: creationTimestamp: "2026-07-02T14:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:55Z" name: model-chainer-7d6d5bb5c4.18be7f4de9d48fe2 namespace: kserve-ci-e2e-test resourceVersion: "15541" uid: 3c67a500-4215-4591-9fbc-1066c0f4e5d8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15532" uid: 6da8acce-17b8-4e99-b88a-defabe0f4db4 kind: Event lastTimestamp: "2026-07-02T14:27:55Z" message: Scaled up replica set model-chainer-7d6d5bb5c4 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:55Z" name: model-chainer.18be7f4de850a314 namespace: kserve-ci-e2e-test resourceVersion: "15535" uid: 9cbf1f16-199d-424c-a060-d5b3b4d4c372 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:55Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15551" uid: 8698d250-8b0c-4ffd-8edb-43cc61c158e3 kind: Event lastTimestamp: "2026-07-02T14:27:55Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-02T14:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:27:55Z" name: model-chainer.18be7f4def782954 namespace: kserve-ci-e2e-test resourceVersion: "15553" uid: 87c438f8-5d64-4c35-9d49-145d810a43b1 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:38:30.259684Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8937c-7586b86695-n6mlq namespace: kserve-ci-e2e-test resourceVersion: "20907" uid: cb34f367-3b5f-4a8f-a70e-1db545a1ca9a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-8937c-7586b86695-n6mlq to ip-10-0-143-249.ec2.internal metadata: creationTimestamp: "2026-07-02T14:38:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:38:30Z" name: sequence-graph-8937c-7586b86695-n6mlq.18be7fe1cc72147b namespace: kserve-ci-e2e-test resourceVersion: "20912" uid: b47298d0-3e71-482e-8639-67ac8a8b7461 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:30Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8937c-7586b86695-n6mlq namespace: kserve-ci-e2e-test resourceVersion: "20918" uid: cb34f367-3b5f-4a8f-a70e-1db545a1ca9a kind: Event lastTimestamp: "2026-07-02T14:38:30Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:38:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:38:30Z" name: sequence-graph-8937c-7586b86695-n6mlq.18be7fe1e6834de0 namespace: kserve-ci-e2e-test resourceVersion: "20929" uid: 99111dbd-c4e7-4922-a021-25f972541827 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8937c} kind: Pod name: sequence-graph-8937c-7586b86695-n6mlq namespace: kserve-ci-e2e-test resourceVersion: "20910" uid: cb34f367-3b5f-4a8f-a70e-1db545a1ca9a kind: Event lastTimestamp: "2026-07-02T14:38:30Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:e0bdb71677d5eb7ee4784378e591e8126d0dcafbbe991e5dcf178c67e85889cd" already present on machine metadata: creationTimestamp: "2026-07-02T14:38:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:30Z" name: sequence-graph-8937c-7586b86695-n6mlq.18be7fe1e7d62058 namespace: kserve-ci-e2e-test resourceVersion: "20931" uid: 989cdf57-cbc1-4edb-a026-6b5832bfd684 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8937c} kind: Pod name: sequence-graph-8937c-7586b86695-n6mlq namespace: kserve-ci-e2e-test resourceVersion: "20910" uid: cb34f367-3b5f-4a8f-a70e-1db545a1ca9a kind: Event lastTimestamp: "2026-07-02T14:38:30Z" message: 'Created container: sequence-graph-8937c' metadata: creationTimestamp: "2026-07-02T14:38:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:30Z" name: sequence-graph-8937c-7586b86695-n6mlq.18be7fe1ea7282d3 namespace: kserve-ci-e2e-test resourceVersion: "20932" uid: 0b8f36a8-4359-42b1-9e6b-4e2c5d311b85 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8937c} kind: Pod name: sequence-graph-8937c-7586b86695-n6mlq namespace: kserve-ci-e2e-test resourceVersion: "20910" uid: cb34f367-3b5f-4a8f-a70e-1db545a1ca9a kind: Event lastTimestamp: "2026-07-02T14:38:30Z" message: Started container sequence-graph-8937c metadata: creationTimestamp: "2026-07-02T14:38:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:30Z" name: sequence-graph-8937c-7586b86695-n6mlq.18be7fe1eae30d6b namespace: kserve-ci-e2e-test resourceVersion: "20933" uid: 137c3f76-da59-4edf-bcb0-eb9269caad04 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8937c} kind: Pod name: sequence-graph-8937c-7586b86695-n6mlq namespace: kserve-ci-e2e-test resourceVersion: "20910" uid: cb34f367-3b5f-4a8f-a70e-1db545a1ca9a kind: Event lastTimestamp: "2026-07-02T14:38:44Z" message: Stopping container sequence-graph-8937c metadata: creationTimestamp: "2026-07-02T14:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:44Z" name: sequence-graph-8937c-7586b86695-n6mlq.18be7fe519942b81 namespace: kserve-ci-e2e-test resourceVersion: "21111" uid: 55584aba-6cb1-4f39-bbcf-fa8a1272161a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:38:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8937c} kind: Pod name: sequence-graph-8937c-7586b86695-n6mlq namespace: kserve-ci-e2e-test resourceVersion: "20910" uid: cb34f367-3b5f-4a8f-a70e-1db545a1ca9a kind: Event lastTimestamp: "2026-07-02T14:39:12Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:38:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:39:12Z" name: sequence-graph-8937c-7586b86695-n6mlq.18be7fe5d92de10b namespace: kserve-ci-e2e-test resourceVersion: "21468" uid: 97ce32d7-18ba-421f-b520-06d205165ecc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-8937c-7586b86695 namespace: kserve-ci-e2e-test resourceVersion: "20902" uid: 07a9dd48-ee2e-4cc2-b23e-fb18dfa03d29 kind: Event lastTimestamp: "2026-07-02T14:38:30Z" message: 'Created pod: sequence-graph-8937c-7586b86695-n6mlq' metadata: creationTimestamp: "2026-07-02T14:38:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:30Z" name: sequence-graph-8937c-7586b86695.18be7fe1cc095c06 namespace: kserve-ci-e2e-test resourceVersion: "20909" uid: 990063b6-ddaa-4d5d-9430-7f3bfb74e278 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-8937c namespace: kserve-ci-e2e-test resourceVersion: "20900" uid: be2324cb-2b08-44ad-9361-75ab749e9490 kind: Event lastTimestamp: "2026-07-02T14:38:30Z" message: Scaled up replica set sequence-graph-8937c-7586b86695 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:38:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:30Z" name: sequence-graph-8937c.18be7fe1cb2582ab namespace: kserve-ci-e2e-test resourceVersion: "20906" uid: c2498df7-edb5-4d62-b90b-fba538eb150c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:30Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-8937c namespace: kserve-ci-e2e-test resourceVersion: "20920" uid: ee6c7eb4-faca-4e3e-93e9-43dccdcda812 kind: Event lastTimestamp: "2026-07-02T14:38:30Z" message: InferenceGraph [sequence-graph-8937c] is Ready metadata: creationTimestamp: "2026-07-02T14:38:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:38:30Z" name: sequence-graph-8937c.18be7fe1cf3390c1 namespace: kserve-ci-e2e-test resourceVersion: "20922" uid: bffe6539-1fc1-43b3-ad4c-1d10929fe476 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:29:05.388199Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-9a7ca-7695d79cf4-h9l2r namespace: kserve-ci-e2e-test resourceVersion: "16365" uid: fa929eaf-731e-484c-902a-ecf4cbdbd504 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-9a7ca-7695d79cf4-h9l2r to ip-10-0-143-249.ec2.internal metadata: creationTimestamp: "2026-07-02T14:29:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:29:05Z" name: sequence-graph-9a7ca-7695d79cf4-h9l2r.18be7f5e477b45fd namespace: kserve-ci-e2e-test resourceVersion: "16370" uid: 71bc770c-aaa8-4ba5-898b-21bd39d04b99 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:05Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-9a7ca-7695d79cf4-h9l2r namespace: kserve-ci-e2e-test resourceVersion: "16368" uid: fa929eaf-731e-484c-902a-ecf4cbdbd504 kind: Event lastTimestamp: "2026-07-02T14:29:05Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-9a7ca-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:29:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:29:05Z" name: sequence-graph-9a7ca-7695d79cf4-h9l2r.18be7f5e4f9aced4 namespace: kserve-ci-e2e-test resourceVersion: "16384" uid: 170d084f-c0b9-4d04-94d1-ee9838396e0b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:06Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-9a7ca-7695d79cf4-h9l2r namespace: kserve-ci-e2e-test resourceVersion: "16375" uid: fa929eaf-731e-484c-902a-ecf4cbdbd504 kind: Event lastTimestamp: "2026-07-02T14:29:06Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:29:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:29:06Z" name: sequence-graph-9a7ca-7695d79cf4-h9l2r.18be7f5e84a9eb92 namespace: kserve-ci-e2e-test resourceVersion: "16397" uid: 89d440f6-d580-4938-9f40-7cb32a437f4b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9a7ca} kind: Pod name: sequence-graph-9a7ca-7695d79cf4-h9l2r namespace: kserve-ci-e2e-test resourceVersion: "16368" uid: fa929eaf-731e-484c-902a-ecf4cbdbd504 kind: Event lastTimestamp: "2026-07-02T14:29:06Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:e0bdb71677d5eb7ee4784378e591e8126d0dcafbbe991e5dcf178c67e85889cd" already present on machine metadata: creationTimestamp: "2026-07-02T14:29:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:29:06Z" name: sequence-graph-9a7ca-7695d79cf4-h9l2r.18be7f5e860eacb2 namespace: kserve-ci-e2e-test resourceVersion: "16399" uid: 38048f00-b704-4ac2-83ea-31dd6037122f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9a7ca} kind: Pod name: sequence-graph-9a7ca-7695d79cf4-h9l2r namespace: kserve-ci-e2e-test resourceVersion: "16368" uid: fa929eaf-731e-484c-902a-ecf4cbdbd504 kind: Event lastTimestamp: "2026-07-02T14:29:06Z" message: 'Created container: sequence-graph-9a7ca' metadata: creationTimestamp: "2026-07-02T14:29:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:29:06Z" name: sequence-graph-9a7ca-7695d79cf4-h9l2r.18be7f5e88f3efe4 namespace: kserve-ci-e2e-test resourceVersion: "16400" uid: 6b8a52e9-7431-4052-a2e4-bbdaee3879fb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9a7ca} kind: Pod name: sequence-graph-9a7ca-7695d79cf4-h9l2r namespace: kserve-ci-e2e-test resourceVersion: "16368" uid: fa929eaf-731e-484c-902a-ecf4cbdbd504 kind: Event lastTimestamp: "2026-07-02T14:29:06Z" message: Started container sequence-graph-9a7ca metadata: creationTimestamp: "2026-07-02T14:29:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:29:06Z" name: sequence-graph-9a7ca-7695d79cf4-h9l2r.18be7f5e89644dab namespace: kserve-ci-e2e-test resourceVersion: "16401" uid: 5085167c-0ffa-4dc2-af99-77b7e32ff41f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9a7ca} kind: Pod name: sequence-graph-9a7ca-7695d79cf4-h9l2r namespace: kserve-ci-e2e-test resourceVersion: "16368" uid: fa929eaf-731e-484c-902a-ecf4cbdbd504 kind: Event lastTimestamp: "2026-07-02T14:37:20Z" message: Stopping container sequence-graph-9a7ca metadata: creationTimestamp: "2026-07-02T14:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:20Z" name: sequence-graph-9a7ca-7695d79cf4-h9l2r.18be7fd175a57a12 namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 517e51bb-2ac9-4c79-b9f4-287707c5f226 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:37:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9a7ca} kind: Pod name: sequence-graph-9a7ca-7695d79cf4-h9l2r namespace: kserve-ci-e2e-test resourceVersion: "16368" uid: fa929eaf-731e-484c-902a-ecf4cbdbd504 kind: Event lastTimestamp: "2026-07-02T14:37:48Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:37:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:48Z" name: sequence-graph-9a7ca-7695d79cf4-h9l2r.18be7fd237ee0138 namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: 0b8744a5-249f-4733-9828-c6a41cdf25fb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-9a7ca-7695d79cf4 namespace: kserve-ci-e2e-test resourceVersion: "16362" uid: 024d3ed8-972d-4f3b-a84b-48065764fb67 kind: Event lastTimestamp: "2026-07-02T14:29:05Z" message: 'Created pod: sequence-graph-9a7ca-7695d79cf4-h9l2r' metadata: creationTimestamp: "2026-07-02T14:29:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:29:05Z" name: sequence-graph-9a7ca-7695d79cf4.18be7f5e46f9246b namespace: kserve-ci-e2e-test resourceVersion: "16367" uid: 2a40f1dd-80e9-4d81-86a0-f3b8249a44d7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-9a7ca namespace: kserve-ci-e2e-test resourceVersion: "16360" uid: cad42273-a2d9-4cd8-88b8-643b0fc39c5a kind: Event lastTimestamp: "2026-07-02T14:29:05Z" message: Scaled up replica set sequence-graph-9a7ca-7695d79cf4 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:29:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:29:05Z" name: sequence-graph-9a7ca.18be7f5e45ea2228 namespace: kserve-ci-e2e-test resourceVersion: "16363" uid: 10984626-9419-441c-b7d7-d8f6ec86c8e5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:05Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-9a7ca namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: c81d0b7a-e110-4faf-8173-c135a33a4bd5 kind: Event lastTimestamp: "2026-07-02T14:29:05Z" message: InferenceGraph [sequence-graph-9a7ca] is Ready metadata: creationTimestamp: "2026-07-02T14:29:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:29:05Z" name: sequence-graph-9a7ca.18be7f5e4cddfe4c namespace: kserve-ci-e2e-test resourceVersion: "16382" uid: 0de6a5a4-09d3-4969-bfa4-a997cfbeee54 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:05Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-9a7ca namespace: kserve-ci-e2e-test resourceVersion: "16355" uid: c81d0b7a-e110-4faf-8173-c135a33a4bd5 kind: Event lastTimestamp: "2026-07-02T14:29:05Z" message: 'Failed to update status for InferenceGraph "sequence-graph-9a7ca": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-9a7ca": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:29:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:29:05Z" name: sequence-graph-9a7ca.18be7f5e4fd9dd0e namespace: kserve-ci-e2e-test resourceVersion: "16385" uid: 9e49b8f6-1046-4185-9820-1baf852aa616 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:29:05Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-9a7ca namespace: kserve-ci-e2e-test resourceVersion: "16355" uid: c81d0b7a-e110-4faf-8173-c135a33a4bd5 kind: Event lastTimestamp: "2026-07-02T14:29:05Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-9a7ca": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:29:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:29:05Z" name: sequence-graph-9a7ca.18be7f5e4fda2150 namespace: kserve-ci-e2e-test resourceVersion: "16386" uid: 33d2fe4e-90cd-4483-ace4-1d2e06a72732 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:29:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-9a7ca namespace: kserve-ci-e2e-test resourceVersion: "16361" uid: 93dc9f80-051d-4ab4-a697-3f872908680f kind: Event lastTimestamp: "2026-07-02T14:29:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:29:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:29:35Z" name: sequence-graph-9a7ca.18be7f61c51087a3 namespace: kserve-ci-e2e-test resourceVersion: "16609" uid: 511c4f58-cf88-4469-becb-89d2f45a9097 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:29:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-9a7ca namespace: kserve-ci-e2e-test resourceVersion: "16361" uid: 93dc9f80-051d-4ab4-a697-3f872908680f kind: Event lastTimestamp: "2026-07-02T14:29:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:29:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:29:35Z" name: sequence-graph-9a7ca.18be7f61c56c0b5f namespace: kserve-ci-e2e-test resourceVersion: "16610" uid: 5f82ce59-5331-4c67-9418-d3cb9531f32b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:39:44.668597Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c3f32-d7476f4ff-9nl99 namespace: kserve-ci-e2e-test resourceVersion: "21729" uid: 828bdff3-9b07-4549-bf67-d8c99187a2f9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-c3f32-d7476f4ff-9nl99 to ip-10-0-143-249.ec2.internal metadata: creationTimestamp: "2026-07-02T14:39:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:39:44Z" name: sequence-graph-c3f32-d7476f4ff-9nl99.18be7ff31f8ff6b3 namespace: kserve-ci-e2e-test resourceVersion: "21734" uid: 09c6d61a-df03-4e0d-8f0c-68abbbdaff11 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:39:45Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c3f32-d7476f4ff-9nl99 namespace: kserve-ci-e2e-test resourceVersion: "21738" uid: 828bdff3-9b07-4549-bf67-d8c99187a2f9 kind: Event lastTimestamp: "2026-07-02T14:39:45Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:39:45Z" name: sequence-graph-c3f32-d7476f4ff-9nl99.18be7ff338f62358 namespace: kserve-ci-e2e-test resourceVersion: "21752" uid: bbf8b69c-d9fc-484e-be77-497e29b6013d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c3f32} kind: Pod name: sequence-graph-c3f32-d7476f4ff-9nl99 namespace: kserve-ci-e2e-test resourceVersion: "21732" uid: 828bdff3-9b07-4549-bf67-d8c99187a2f9 kind: Event lastTimestamp: "2026-07-02T14:39:45Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:e0bdb71677d5eb7ee4784378e591e8126d0dcafbbe991e5dcf178c67e85889cd" already present on machine metadata: creationTimestamp: "2026-07-02T14:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:39:45Z" name: sequence-graph-c3f32-d7476f4ff-9nl99.18be7ff33a3ab797 namespace: kserve-ci-e2e-test resourceVersion: "21754" uid: 64beded4-3b47-4ffe-9880-1e581c3e5567 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c3f32} kind: Pod name: sequence-graph-c3f32-d7476f4ff-9nl99 namespace: kserve-ci-e2e-test resourceVersion: "21732" uid: 828bdff3-9b07-4549-bf67-d8c99187a2f9 kind: Event lastTimestamp: "2026-07-02T14:39:45Z" message: 'Created container: sequence-graph-c3f32' metadata: creationTimestamp: "2026-07-02T14:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:39:45Z" name: sequence-graph-c3f32-d7476f4ff-9nl99.18be7ff33d5d8f79 namespace: kserve-ci-e2e-test resourceVersion: "21755" uid: ed297e5d-3ac7-47d0-bbf7-c6c26f8355b6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c3f32} kind: Pod name: sequence-graph-c3f32-d7476f4ff-9nl99 namespace: kserve-ci-e2e-test resourceVersion: "21732" uid: 828bdff3-9b07-4549-bf67-d8c99187a2f9 kind: Event lastTimestamp: "2026-07-02T14:39:45Z" message: Started container sequence-graph-c3f32 metadata: creationTimestamp: "2026-07-02T14:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:39:45Z" name: sequence-graph-c3f32-d7476f4ff-9nl99.18be7ff33dcd1e1a namespace: kserve-ci-e2e-test resourceVersion: "21756" uid: 97207e80-7bea-451c-88d0-6f6d2b36f099 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c3f32} kind: Pod name: sequence-graph-c3f32-d7476f4ff-9nl99 namespace: kserve-ci-e2e-test resourceVersion: "21732" uid: 828bdff3-9b07-4549-bf67-d8c99187a2f9 kind: Event lastTimestamp: "2026-07-02T14:47:59Z" message: Stopping container sequence-graph-c3f32 metadata: creationTimestamp: "2026-07-02T14:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:47:59Z" name: sequence-graph-c3f32-d7476f4ff-9nl99.18be8066471dd6c2 namespace: kserve-ci-e2e-test resourceVersion: "25413" uid: c4211e79-ad97-4965-9ca8-2cd05c70a278 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:48:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c3f32} kind: Pod name: sequence-graph-c3f32-d7476f4ff-9nl99 namespace: kserve-ci-e2e-test resourceVersion: "21732" uid: 828bdff3-9b07-4549-bf67-d8c99187a2f9 kind: Event lastTimestamp: "2026-07-02T14:48:26Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:26Z" name: sequence-graph-c3f32-d7476f4ff-9nl99.18be8066e85ecf65 namespace: kserve-ci-e2e-test resourceVersion: "25923" uid: 0aa03711-d944-45a5-be0d-e26ae8131969 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:39:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-c3f32-d7476f4ff namespace: kserve-ci-e2e-test resourceVersion: "21725" uid: 7f602ff1-8e90-4f46-b574-eb6843dde871 kind: Event lastTimestamp: "2026-07-02T14:39:44Z" message: 'Created pod: sequence-graph-c3f32-d7476f4ff-9nl99' metadata: creationTimestamp: "2026-07-02T14:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:39:44Z" name: sequence-graph-c3f32-d7476f4ff.18be7ff31f1874eb namespace: kserve-ci-e2e-test resourceVersion: "21731" uid: f07bdee3-acff-4893-8c4f-e98dea9133d2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:39:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-c3f32 namespace: kserve-ci-e2e-test resourceVersion: "21721" uid: f9c21b09-318d-4269-af34-988a185e09cc kind: Event lastTimestamp: "2026-07-02T14:39:44Z" message: Scaled up replica set sequence-graph-c3f32-d7476f4ff from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:39:44Z" name: sequence-graph-c3f32.18be7ff31e43b1ab namespace: kserve-ci-e2e-test resourceVersion: "21727" uid: 007307d1-dc4e-4d03-8221-5210b3668f53 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:39:44Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-c3f32 namespace: kserve-ci-e2e-test resourceVersion: "21741" uid: a5133849-7a5f-443f-99ec-7b2f3a9f4636 kind: Event lastTimestamp: "2026-07-02T14:39:44Z" message: InferenceGraph [sequence-graph-c3f32] is Ready metadata: creationTimestamp: "2026-07-02T14:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:39:44Z" name: sequence-graph-c3f32.18be7ff321e45388 namespace: kserve-ci-e2e-test resourceVersion: "21743" uid: d540f4a9-b253-43ac-977b-414f703339f7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:39:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c3f32 namespace: kserve-ci-e2e-test resourceVersion: "21724" uid: 86e3ef2e-1ec2-46bf-be84-5a0c66700efc kind: Event lastTimestamp: "2026-07-02T14:39:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:39:59Z" name: sequence-graph-c3f32.18be7ff69cd273de namespace: kserve-ci-e2e-test resourceVersion: "21864" uid: d3ad2452-d914-4007-9f3f-83f239185b02 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:39:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c3f32 namespace: kserve-ci-e2e-test resourceVersion: "21724" uid: 86e3ef2e-1ec2-46bf-be84-5a0c66700efc kind: Event lastTimestamp: "2026-07-02T14:39:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:39:59Z" name: sequence-graph-c3f32.18be7ff69d1e524c namespace: kserve-ci-e2e-test resourceVersion: "21865" uid: b0b14bb2-998c-47ec-9e02-728dd03b790e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:40:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c3f32 namespace: kserve-ci-e2e-test resourceVersion: "21862" uid: 86e3ef2e-1ec2-46bf-be84-5a0c66700efc kind: Event lastTimestamp: "2026-07-02T14:40:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:40:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:40:14Z" name: sequence-graph-c3f32.18be7ffa1baff903 namespace: kserve-ci-e2e-test resourceVersion: "21971" uid: e4979457-2dfb-4541-b399-21389e4b2453 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:40:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c3f32 namespace: kserve-ci-e2e-test resourceVersion: "21862" uid: 86e3ef2e-1ec2-46bf-be84-5a0c66700efc kind: Event lastTimestamp: "2026-07-02T14:40:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:40:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:40:14Z" name: sequence-graph-c3f32.18be7ffa1c2093c2 namespace: kserve-ci-e2e-test resourceVersion: "21974" uid: 1302fa29-9ea5-4e33-ad01-d0769c452137 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:49:10.002192Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-2963e-85db8f6f9d-gw7pz namespace: kserve-ci-e2e-test resourceVersion: "26636" uid: 12591e02-4ff0-4088-a53f-c213b0997549 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-2963e-85db8f6f9d-gw7pz to ip-10-0-143-249.ec2.internal metadata: creationTimestamp: "2026-07-02T14:49:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:49:10Z" name: splitter-graph-2963e-85db8f6f9d-gw7pz.18be8076c0120c82 namespace: kserve-ci-e2e-test resourceVersion: "26643" uid: 5b1c8fff-f673-442c-8e9e-d9a802fd6fba reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:49:10Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-2963e-85db8f6f9d-gw7pz namespace: kserve-ci-e2e-test resourceVersion: "26647" uid: 12591e02-4ff0-4088-a53f-c213b0997549 kind: Event lastTimestamp: "2026-07-02T14:49:10Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:49:10Z" name: splitter-graph-2963e-85db8f6f9d-gw7pz.18be8076d98fc7c8 namespace: kserve-ci-e2e-test resourceVersion: "26660" uid: 1b9955a0-d713-4d67-aa43-86f434028dbf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-2963e} kind: Pod name: splitter-graph-2963e-85db8f6f9d-gw7pz namespace: kserve-ci-e2e-test resourceVersion: "26639" uid: 12591e02-4ff0-4088-a53f-c213b0997549 kind: Event lastTimestamp: "2026-07-02T14:49:10Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:e0bdb71677d5eb7ee4784378e591e8126d0dcafbbe991e5dcf178c67e85889cd" already present on machine metadata: creationTimestamp: "2026-07-02T14:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:49:10Z" name: splitter-graph-2963e-85db8f6f9d-gw7pz.18be8076dad8dc38 namespace: kserve-ci-e2e-test resourceVersion: "26662" uid: 00bec21b-4384-458f-a8b0-92623674c7cb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-2963e} kind: Pod name: splitter-graph-2963e-85db8f6f9d-gw7pz namespace: kserve-ci-e2e-test resourceVersion: "26639" uid: 12591e02-4ff0-4088-a53f-c213b0997549 kind: Event lastTimestamp: "2026-07-02T14:49:10Z" message: 'Created container: splitter-graph-2963e' metadata: creationTimestamp: "2026-07-02T14:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:49:10Z" name: splitter-graph-2963e-85db8f6f9d-gw7pz.18be8076dd7046a5 namespace: kserve-ci-e2e-test resourceVersion: "26663" uid: 271297b8-d45c-4e50-a001-bc8e6c9b2ed1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-2963e} kind: Pod name: splitter-graph-2963e-85db8f6f9d-gw7pz namespace: kserve-ci-e2e-test resourceVersion: "26639" uid: 12591e02-4ff0-4088-a53f-c213b0997549 kind: Event lastTimestamp: "2026-07-02T14:49:10Z" message: Started container splitter-graph-2963e metadata: creationTimestamp: "2026-07-02T14:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:49:10Z" name: splitter-graph-2963e-85db8f6f9d-gw7pz.18be8076dde34947 namespace: kserve-ci-e2e-test resourceVersion: "26664" uid: 9c4e7879-4de4-4237-98d6-d6f8d9a24bc4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:57:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-2963e} kind: Pod name: splitter-graph-2963e-85db8f6f9d-gw7pz namespace: kserve-ci-e2e-test resourceVersion: "26639" uid: 12591e02-4ff0-4088-a53f-c213b0997549 kind: Event lastTimestamp: "2026-07-02T14:57:24Z" message: Stopping container splitter-graph-2963e metadata: creationTimestamp: "2026-07-02T14:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:57:24Z" name: splitter-graph-2963e-85db8f6f9d-gw7pz.18be80e9ea381c1b namespace: kserve-ci-e2e-test resourceVersion: "30131" uid: 07807703-1c06-43b9-9a2c-c206f00371fc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:57:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-2963e} kind: Pod name: splitter-graph-2963e-85db8f6f9d-gw7pz namespace: kserve-ci-e2e-test resourceVersion: "26639" uid: 12591e02-4ff0-4088-a53f-c213b0997549 kind: Event lastTimestamp: "2026-07-02T14:57:52Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:57:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:57:52Z" name: splitter-graph-2963e-85db8f6f9d-gw7pz.18be80ea8951ec4c namespace: kserve-ci-e2e-test resourceVersion: "30369" uid: c590e6aa-2388-4a75-87a4-659661587619 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:49:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-2963e-85db8f6f9d namespace: kserve-ci-e2e-test resourceVersion: "26632" uid: 5cb61bc2-500f-40d4-9717-f032026bff88 kind: Event lastTimestamp: "2026-07-02T14:49:09Z" message: 'Created pod: splitter-graph-2963e-85db8f6f9d-gw7pz' metadata: creationTimestamp: "2026-07-02T14:49:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:49:09Z" name: splitter-graph-2963e-85db8f6f9d.18be8076bfa086cb namespace: kserve-ci-e2e-test resourceVersion: "26638" uid: aca75a6d-07b3-4d47-b3ee-e02c175b09db reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:49:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-2963e namespace: kserve-ci-e2e-test resourceVersion: "26630" uid: 3e302f7d-a558-4e58-90c7-b9a7249c3bf4 kind: Event lastTimestamp: "2026-07-02T14:49:09Z" message: Scaled up replica set splitter-graph-2963e-85db8f6f9d from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:49:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:49:09Z" name: splitter-graph-2963e.18be8076bef1ee80 namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: 58c40594-25bc-470d-8331-f4a81f400ad2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:49:10Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-2963e namespace: kserve-ci-e2e-test resourceVersion: "26649" uid: 4812cf2b-cd48-47d8-a6f3-567547473fda kind: Event lastTimestamp: "2026-07-02T14:49:10Z" message: InferenceGraph [splitter-graph-2963e] is Ready metadata: creationTimestamp: "2026-07-02T14:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:49:10Z" name: splitter-graph-2963e.18be8076c3f9e527 namespace: kserve-ci-e2e-test resourceVersion: "26651" uid: 5b88a40d-1f79-4ebc-a4f1-e3a0f5f8c007 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:49:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-2963e namespace: kserve-ci-e2e-test resourceVersion: "26631" uid: b0cc28a2-0dc5-4cd4-be47-1940c41fc629 kind: Event lastTimestamp: "2026-07-02T14:49:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:49:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:49:40Z" name: splitter-graph-2963e.18be807a3d6e6646 namespace: kserve-ci-e2e-test resourceVersion: "26903" uid: 5b50ffd1-07b4-4b3e-8ce7-4e4d7f46b995 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:49:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-2963e namespace: kserve-ci-e2e-test resourceVersion: "26631" uid: b0cc28a2-0dc5-4cd4-be47-1940c41fc629 kind: Event lastTimestamp: "2026-07-02T14:49:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:49:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:49:40Z" name: splitter-graph-2963e.18be807a3ddbb936 namespace: kserve-ci-e2e-test resourceVersion: "26904" uid: 2b0be578-26c9-4316-9c09-c1ee341a93f0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:47:49.707131Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-484ca-db8ddbcc6-fjb5m namespace: kserve-ci-e2e-test resourceVersion: "25300" uid: 54a22088-ebd6-43b6-bd9b-897d5f5eda7c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-484ca-db8ddbcc6-fjb5m to ip-10-0-143-249.ec2.internal metadata: creationTimestamp: "2026-07-02T14:47:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:47:49Z" name: splitter-graph-484ca-db8ddbcc6-fjb5m.18be80640e1ca7da namespace: kserve-ci-e2e-test resourceVersion: "25305" uid: 70531574-a2d0-46b7-b0f9-03e1a5b9b73f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:49Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-484ca-db8ddbcc6-fjb5m namespace: kserve-ci-e2e-test resourceVersion: "25303" uid: 54a22088-ebd6-43b6-bd9b-897d5f5eda7c kind: Event lastTimestamp: "2026-07-02T14:47:49Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-484ca-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:47:49Z" name: splitter-graph-484ca-db8ddbcc6-fjb5m.18be80641a53a772 namespace: kserve-ci-e2e-test resourceVersion: "25318" uid: 36e97491-b21e-4bb8-b32a-1c7c179f1e75 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:50Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-484ca-db8ddbcc6-fjb5m namespace: kserve-ci-e2e-test resourceVersion: "25309" uid: 54a22088-ebd6-43b6-bd9b-897d5f5eda7c kind: Event lastTimestamp: "2026-07-02T14:47:50Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:47:50Z" name: splitter-graph-484ca-db8ddbcc6-fjb5m.18be80644b23726c namespace: kserve-ci-e2e-test resourceVersion: "25327" uid: e2421bd3-d230-46bb-9bb6-a09356e4c14f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-484ca} kind: Pod name: splitter-graph-484ca-db8ddbcc6-fjb5m namespace: kserve-ci-e2e-test resourceVersion: "25303" uid: 54a22088-ebd6-43b6-bd9b-897d5f5eda7c kind: Event lastTimestamp: "2026-07-02T14:47:50Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:e0bdb71677d5eb7ee4784378e591e8126d0dcafbbe991e5dcf178c67e85889cd" already present on machine metadata: creationTimestamp: "2026-07-02T14:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:47:50Z" name: splitter-graph-484ca-db8ddbcc6-fjb5m.18be80644c72cb16 namespace: kserve-ci-e2e-test resourceVersion: "25329" uid: ae9951b3-4923-4b15-a1f2-37a61120c7de reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-484ca} kind: Pod name: splitter-graph-484ca-db8ddbcc6-fjb5m namespace: kserve-ci-e2e-test resourceVersion: "25303" uid: 54a22088-ebd6-43b6-bd9b-897d5f5eda7c kind: Event lastTimestamp: "2026-07-02T14:47:50Z" message: 'Created container: splitter-graph-484ca' metadata: creationTimestamp: "2026-07-02T14:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:47:50Z" name: splitter-graph-484ca-db8ddbcc6-fjb5m.18be80644fddbc16 namespace: kserve-ci-e2e-test resourceVersion: "25330" uid: c7b420ed-f6c4-476f-9137-a8b621c2b739 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-484ca} kind: Pod name: splitter-graph-484ca-db8ddbcc6-fjb5m namespace: kserve-ci-e2e-test resourceVersion: "25303" uid: 54a22088-ebd6-43b6-bd9b-897d5f5eda7c kind: Event lastTimestamp: "2026-07-02T14:47:50Z" message: Started container splitter-graph-484ca metadata: creationTimestamp: "2026-07-02T14:47:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:47:50Z" name: splitter-graph-484ca-db8ddbcc6-fjb5m.18be806450499f60 namespace: kserve-ci-e2e-test resourceVersion: "25331" uid: 74ef09e5-827e-4381-b553-ff581ebe9d72 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-484ca} kind: Pod name: splitter-graph-484ca-db8ddbcc6-fjb5m namespace: kserve-ci-e2e-test resourceVersion: "25303" uid: 54a22088-ebd6-43b6-bd9b-897d5f5eda7c kind: Event lastTimestamp: "2026-07-02T14:47:59Z" message: Stopping container splitter-graph-484ca metadata: creationTimestamp: "2026-07-02T14:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:47:59Z" name: splitter-graph-484ca-db8ddbcc6-fjb5m.18be8066666b962e namespace: kserve-ci-e2e-test resourceVersion: "25528" uid: e0a75722-2386-4d25-af25-525457fe0ee0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:48:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-484ca} kind: Pod name: splitter-graph-484ca-db8ddbcc6-fjb5m namespace: kserve-ci-e2e-test resourceVersion: "25303" uid: 54a22088-ebd6-43b6-bd9b-897d5f5eda7c kind: Event lastTimestamp: "2026-07-02T14:48:26Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:26Z" name: splitter-graph-484ca-db8ddbcc6-fjb5m.18be8066e830463e namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: a282303b-6be5-4859-8794-7dc787e02bdf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-484ca-db8ddbcc6 namespace: kserve-ci-e2e-test resourceVersion: "25297" uid: c745ddd6-fe8b-4e6d-a4e1-a64228bb2c21 kind: Event lastTimestamp: "2026-07-02T14:47:49Z" message: 'Created pod: splitter-graph-484ca-db8ddbcc6-fjb5m' metadata: creationTimestamp: "2026-07-02T14:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:47:49Z" name: splitter-graph-484ca-db8ddbcc6.18be80640da54bc8 namespace: kserve-ci-e2e-test resourceVersion: "25301" uid: 2603ede8-ac99-42cb-bb4e-9129bc44f70c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-484ca namespace: kserve-ci-e2e-test resourceVersion: "25295" uid: 753dc404-9a9e-4f30-81f4-e6550ed5f71c kind: Event lastTimestamp: "2026-07-02T14:47:49Z" message: Scaled up replica set splitter-graph-484ca-db8ddbcc6 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:47:49Z" name: splitter-graph-484ca.18be80640cd30239 namespace: kserve-ci-e2e-test resourceVersion: "25298" uid: 9fed1460-6468-4156-a174-e3624c3e4c56 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:49Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-484ca namespace: kserve-ci-e2e-test resourceVersion: "25313" uid: 65df026b-740e-47f1-9964-f7c9c92d6eee kind: Event lastTimestamp: "2026-07-02T14:47:49Z" message: InferenceGraph [splitter-graph-484ca] is Ready metadata: creationTimestamp: "2026-07-02T14:47:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:47:49Z" name: splitter-graph-484ca.18be806412b4b1b9 namespace: kserve-ci-e2e-test resourceVersion: "25315" uid: 59e0293e-471d-420c-bf0f-6d00ec5826a8 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:27:09.467777Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1c2b7-predictor-768bc57fc6-xclr4 namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 942cb8ee-ae7e-412b-a8ed-c2a5b3628019 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1c2b7-predictor-768bc57fc6-xclr4 to ip-10-0-134-54.ec2.internal metadata: creationTimestamp: "2026-07-02T14:27:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:27:09Z" name: success-200-isvc-1c2b7-predictor-768bc57fc6-xclr4.18be7f434a15f318 namespace: kserve-ci-e2e-test resourceVersion: "15043" uid: 8238758c-54cf-4969-ab6c-6af137721d1b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:09Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1c2b7-predictor-768bc57fc6-xclr4 namespace: kserve-ci-e2e-test resourceVersion: "15056" uid: 942cb8ee-ae7e-412b-a8ed-c2a5b3628019 kind: Event lastTimestamp: "2026-07-02T14:27:09Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:27:09Z" name: success-200-isvc-1c2b7-predictor-768bc57fc6-xclr4.18be7f4363b39482 namespace: kserve-ci-e2e-test resourceVersion: "15111" uid: 3883a6c2-72aa-487d-8212-88f87023a36b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1c2b7-predictor-768bc57fc6-xclr4 namespace: kserve-ci-e2e-test resourceVersion: "15040" uid: 942cb8ee-ae7e-412b-a8ed-c2a5b3628019 kind: Event lastTimestamp: "2026-07-02T14:27:09Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:09Z" name: success-200-isvc-1c2b7-predictor-768bc57fc6-xclr4.18be7f4365341a5e namespace: kserve-ci-e2e-test resourceVersion: "15113" uid: a18171d5-1650-47ef-a767-877b2697ce9f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1c2b7-predictor-768bc57fc6-xclr4 namespace: kserve-ci-e2e-test resourceVersion: "15040" uid: 942cb8ee-ae7e-412b-a8ed-c2a5b3628019 kind: Event lastTimestamp: "2026-07-02T14:27:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:10Z" name: success-200-isvc-1c2b7-predictor-768bc57fc6-xclr4.18be7f4369f2a7ab namespace: kserve-ci-e2e-test resourceVersion: "15116" uid: c482ba65-7e93-4847-b05b-441de7bcd1f9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1c2b7-predictor-768bc57fc6-xclr4 namespace: kserve-ci-e2e-test resourceVersion: "15040" uid: 942cb8ee-ae7e-412b-a8ed-c2a5b3628019 kind: Event lastTimestamp: "2026-07-02T14:27:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:10Z" name: success-200-isvc-1c2b7-predictor-768bc57fc6-xclr4.18be7f436a6a1c2a namespace: kserve-ci-e2e-test resourceVersion: "15117" uid: 58b6c0e9-5ffe-4cfa-b8a7-31eff048d4d1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1c2b7-predictor-768bc57fc6-xclr4 namespace: kserve-ci-e2e-test resourceVersion: "15040" uid: 942cb8ee-ae7e-412b-a8ed-c2a5b3628019 kind: Event lastTimestamp: "2026-07-02T14:27:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:10Z" name: success-200-isvc-1c2b7-predictor-768bc57fc6-xclr4.18be7f436aa16769 namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: a4131702-a942-46e6-905e-ee842b50f3cb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1c2b7-predictor-768bc57fc6-xclr4 namespace: kserve-ci-e2e-test resourceVersion: "15040" uid: 942cb8ee-ae7e-412b-a8ed-c2a5b3628019 kind: Event lastTimestamp: "2026-07-02T14:27:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:10Z" name: success-200-isvc-1c2b7-predictor-768bc57fc6-xclr4.18be7f436dff3edf namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: c580fdf7-69f9-4ba3-8778-4e6fc450fac1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1c2b7-predictor-768bc57fc6-xclr4 namespace: kserve-ci-e2e-test resourceVersion: "15040" uid: 942cb8ee-ae7e-412b-a8ed-c2a5b3628019 kind: Event lastTimestamp: "2026-07-02T14:27:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:10Z" name: success-200-isvc-1c2b7-predictor-768bc57fc6-xclr4.18be7f436e7a627a namespace: kserve-ci-e2e-test resourceVersion: "15123" uid: d4fe37bb-cf5a-4ce2-80d6-dc65247c8948 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T14:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1c2b7-predictor-768bc57fc6-xclr4 namespace: kserve-ci-e2e-test resourceVersion: "15040" uid: 942cb8ee-ae7e-412b-a8ed-c2a5b3628019 kind: Event lastTimestamp: "2026-07-02T14:36:26Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:26Z" name: success-200-isvc-1c2b7-predictor-768bc57fc6-xclr4.18be7f43772b344b namespace: kserve-ci-e2e-test resourceVersion: "19544" uid: c59c2e3f-7184-471e-b0bb-133a39ba4d3f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1c2b7-predictor-768bc57fc6-xclr4 namespace: kserve-ci-e2e-test resourceVersion: "15040" uid: 942cb8ee-ae7e-412b-a8ed-c2a5b3628019 kind: Event lastTimestamp: "2026-07-02T14:36:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:24Z" name: success-200-isvc-1c2b7-predictor-768bc57fc6-xclr4.18be7fc47feddc22 namespace: kserve-ci-e2e-test resourceVersion: "19441" uid: 8b094b1a-4f2f-4f02-99a6-4da8b57fd06c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1c2b7-predictor-768bc57fc6-xclr4 namespace: kserve-ci-e2e-test resourceVersion: "15040" uid: 942cb8ee-ae7e-412b-a8ed-c2a5b3628019 kind: Event lastTimestamp: "2026-07-02T14:36:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:24Z" name: success-200-isvc-1c2b7-predictor-768bc57fc6-xclr4.18be7fc47fee732e namespace: kserve-ci-e2e-test resourceVersion: "19443" uid: 620ff91b-a464-450f-810c-644d0117a501 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1c2b7-predictor-768bc57fc6-xclr4 namespace: kserve-ci-e2e-test resourceVersion: "15040" uid: 942cb8ee-ae7e-412b-a8ed-c2a5b3628019 kind: Event lastTimestamp: "2026-07-02T14:36:26Z" message: 'Readiness probe failed: Get "https://10.133.0.24:8643/healthz": dial tcp 10.133.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:26Z" name: success-200-isvc-1c2b7-predictor-768bc57fc6-xclr4.18be7fc4ebb332fa namespace: kserve-ci-e2e-test resourceVersion: "19541" uid: ef366fa8-05df-44d0-b72b-24d6382641c8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1c2b7-predictor-768bc57fc6 namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: 68db69d5-674f-4a5f-a38b-302d35b20e3f kind: Event lastTimestamp: "2026-07-02T14:27:09Z" message: 'Created pod: success-200-isvc-1c2b7-predictor-768bc57fc6-xclr4' metadata: creationTimestamp: "2026-07-02T14:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:09Z" name: success-200-isvc-1c2b7-predictor-768bc57fc6.18be7f43499ef2a1 namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: 7005a665-fd29-421d-8c15-b06fbfd46fd0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1c2b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: c41cb58b-0faa-4147-b984-66e4a3724b7c kind: Event lastTimestamp: "2026-07-02T14:27:09Z" message: Scaled up replica set success-200-isvc-1c2b7-predictor-768bc57fc6 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:09Z" name: success-200-isvc-1c2b7-predictor.18be7f434656e3f6 namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: eefc5206-8f29-4e8e-b0a2-a6201399e226 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:27:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1c2b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "15021" uid: dd8b5310-2c41-4943-8c17-a829dd4376a6 kind: Event lastTimestamp: "2026-07-02T14:27:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:27:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:39Z" name: success-200-isvc-1c2b7-predictor.18be7f46c4cd644f namespace: kserve-ci-e2e-test resourceVersion: "15402" uid: c599ccdb-83c4-4b40-bab9-7e16be1e975e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:27:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1c2b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "15021" uid: dd8b5310-2c41-4943-8c17-a829dd4376a6 kind: Event lastTimestamp: "2026-07-02T14:27:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:27:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:39Z" name: success-200-isvc-1c2b7-predictor.18be7f46c53b8732 namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 6b9b0ed4-3093-4050-abe5-b979fbf13ad9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:27:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1c2b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "15267" uid: dd8b5310-2c41-4943-8c17-a829dd4376a6 kind: Event lastTimestamp: "2026-07-02T14:28:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:27:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:39Z" name: success-200-isvc-1c2b7-predictor.18be7f4dc2bd18ee namespace: kserve-ci-e2e-test resourceVersion: "16148" uid: 9c296c0b-ccb1-42a4-bd31-49772e002819 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:27:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1c2b7-predictor namespace: kserve-ci-e2e-test resourceVersion: "15267" uid: dd8b5310-2c41-4943-8c17-a829dd4376a6 kind: Event lastTimestamp: "2026-07-02T14:28:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:27:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:39Z" name: success-200-isvc-1c2b7-predictor.18be7f4dc34603e9 namespace: kserve-ci-e2e-test resourceVersion: "16149" uid: 709d8e5e-b572-4138-80eb-fdd716927510 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:27:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1c2b7 namespace: kserve-ci-e2e-test resourceVersion: "15011" uid: ea0df938-b7ed-41dc-a40e-7dee629bc6b8 kind: Event lastTimestamp: "2026-07-02T14:27:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:27:56Z" name: success-200-isvc-1c2b7.18be7f43466d7b9f namespace: kserve-ci-e2e-test resourceVersion: "15590" uid: 2b81b3f2-247a-47a9-b057-2fca29a456da reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1c2b7 namespace: kserve-ci-e2e-test resourceVersion: "15011" uid: ea0df938-b7ed-41dc-a40e-7dee629bc6b8 kind: Event lastTimestamp: "2026-07-02T14:27:09Z" message: 'Failed to update status for InferenceService "success-200-isvc-1c2b7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1c2b7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:27:09Z" name: success-200-isvc-1c2b7.18be7f4346d58161 namespace: kserve-ci-e2e-test resourceVersion: "15026" uid: 03dfa474-1961-48bf-b68f-e3e87b916b5e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1c2b7 namespace: kserve-ci-e2e-test resourceVersion: "15011" uid: ea0df938-b7ed-41dc-a40e-7dee629bc6b8 kind: Event lastTimestamp: "2026-07-02T14:27:09Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1c2b7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:27:09Z" name: success-200-isvc-1c2b7.18be7f4346d5ce6f namespace: kserve-ci-e2e-test resourceVersion: "15029" uid: 276179ef-65ac-4e74-8578-e28de1c23e30 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1c2b7 namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: ea0df938-b7ed-41dc-a40e-7dee629bc6b8 kind: Event lastTimestamp: "2026-07-02T14:27:56Z" message: InferenceService [success-200-isvc-1c2b7] is Ready metadata: creationTimestamp: "2026-07-02T14:27:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:27:56Z" name: success-200-isvc-1c2b7.18be7f4e365aa21d namespace: kserve-ci-e2e-test resourceVersion: "15588" uid: 9d30d1da-4535-4a92-9b1b-e3de0f66c418 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:47:59.934978Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2963e-predictor-5f9d949bd-h2vcz namespace: kserve-ci-e2e-test resourceVersion: "25575" uid: 389a18ea-e78b-461f-a2ec-b5071b01e38f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-2963e-predictor-5f9d949bd-h2vcz to ip-10-0-134-54.ec2.internal metadata: creationTimestamp: "2026-07-02T14:47:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:47:59Z" name: success-200-isvc-2963e-predictor-5f9d949bd-h2vcz.18be80666fbd2ec4 namespace: kserve-ci-e2e-test resourceVersion: "25580" uid: c8ee1da9-e7f6-4f58-81e3-b7280e6f01a1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2963e-predictor-5f9d949bd-h2vcz namespace: kserve-ci-e2e-test resourceVersion: "25590" uid: 389a18ea-e78b-461f-a2ec-b5071b01e38f kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:48:00Z" name: success-200-isvc-2963e-predictor-5f9d949bd-h2vcz.18be80668bbaf59b namespace: kserve-ci-e2e-test resourceVersion: "25637" uid: b38730c2-b2e6-4130-afa0-f9cbb1d283bd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2963e-predictor-5f9d949bd-h2vcz namespace: kserve-ci-e2e-test resourceVersion: "25578" uid: 389a18ea-e78b-461f-a2ec-b5071b01e38f kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:00Z" name: success-200-isvc-2963e-predictor-5f9d949bd-h2vcz.18be80668d403e6a namespace: kserve-ci-e2e-test resourceVersion: "25640" uid: 6f0abdf2-c784-42ae-8edf-c27dc4697efc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2963e-predictor-5f9d949bd-h2vcz namespace: kserve-ci-e2e-test resourceVersion: "25578" uid: 389a18ea-e78b-461f-a2ec-b5071b01e38f kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:00Z" name: success-200-isvc-2963e-predictor-5f9d949bd-h2vcz.18be806691d4df74 namespace: kserve-ci-e2e-test resourceVersion: "25645" uid: d3226d83-1cc4-4930-bd6c-d280fbd6f3d6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2963e-predictor-5f9d949bd-h2vcz namespace: kserve-ci-e2e-test resourceVersion: "25578" uid: 389a18ea-e78b-461f-a2ec-b5071b01e38f kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:00Z" name: success-200-isvc-2963e-predictor-5f9d949bd-h2vcz.18be806692852ebd namespace: kserve-ci-e2e-test resourceVersion: "25646" uid: 5e44efb3-32c6-4878-89e1-37eb1c528eb6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2963e-predictor-5f9d949bd-h2vcz namespace: kserve-ci-e2e-test resourceVersion: "25578" uid: 389a18ea-e78b-461f-a2ec-b5071b01e38f kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:00Z" name: success-200-isvc-2963e-predictor-5f9d949bd-h2vcz.18be80669290e795 namespace: kserve-ci-e2e-test resourceVersion: "25647" uid: 7e7ff049-0def-40f5-b377-7a5c48c180c2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2963e-predictor-5f9d949bd-h2vcz namespace: kserve-ci-e2e-test resourceVersion: "25578" uid: 389a18ea-e78b-461f-a2ec-b5071b01e38f kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:00Z" name: success-200-isvc-2963e-predictor-5f9d949bd-h2vcz.18be8066953280f5 namespace: kserve-ci-e2e-test resourceVersion: "25648" uid: f28892a8-9f9c-432a-9f25-ad4bfb2ef36b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2963e-predictor-5f9d949bd-h2vcz namespace: kserve-ci-e2e-test resourceVersion: "25578" uid: 389a18ea-e78b-461f-a2ec-b5071b01e38f kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:00Z" name: success-200-isvc-2963e-predictor-5f9d949bd-h2vcz.18be806695dee5cc namespace: kserve-ci-e2e-test resourceVersion: "25651" uid: 49487346-a34b-4ad7-8f4b-8756aff19c37 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-02T14:48:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2963e-predictor-5f9d949bd-h2vcz namespace: kserve-ci-e2e-test resourceVersion: "25578" uid: 389a18ea-e78b-461f-a2ec-b5071b01e38f kind: Event lastTimestamp: "2026-07-02T14:57:27Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:57:27Z" name: success-200-isvc-2963e-predictor-5f9d949bd-h2vcz.18be8066b95f658d namespace: kserve-ci-e2e-test resourceVersion: "30183" uid: 4e6b179e-0d5f-4e44-b2cc-ea8250c6df69 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:57:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2963e-predictor-5f9d949bd-h2vcz namespace: kserve-ci-e2e-test resourceVersion: "25578" uid: 389a18ea-e78b-461f-a2ec-b5071b01e38f kind: Event lastTimestamp: "2026-07-02T14:57:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:57:24Z" name: success-200-isvc-2963e-predictor-5f9d949bd-h2vcz.18be80e9f0ff70e6 namespace: kserve-ci-e2e-test resourceVersion: "30148" uid: cd747c86-e071-4d01-bd37-d7ab894e4dfe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:57:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2963e-predictor-5f9d949bd-h2vcz namespace: kserve-ci-e2e-test resourceVersion: "25578" uid: 389a18ea-e78b-461f-a2ec-b5071b01e38f kind: Event lastTimestamp: "2026-07-02T14:57:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:57:24Z" name: success-200-isvc-2963e-predictor-5f9d949bd-h2vcz.18be80e9f0ffeb89 namespace: kserve-ci-e2e-test resourceVersion: "30154" uid: c24970a4-8e69-48f3-828c-e47d46e0ffef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:57:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2963e-predictor-5f9d949bd-h2vcz namespace: kserve-ci-e2e-test resourceVersion: "25578" uid: 389a18ea-e78b-461f-a2ec-b5071b01e38f kind: Event lastTimestamp: "2026-07-02T14:57:27Z" message: 'Readiness probe failed: Get "https://10.133.0.40:8643/healthz": dial tcp 10.133.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:57:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:57:27Z" name: success-200-isvc-2963e-predictor-5f9d949bd-h2vcz.18be80ea820935b7 namespace: kserve-ci-e2e-test resourceVersion: "30182" uid: 92e9342f-cfa8-4737-a30a-b6a137428e26 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-2963e-predictor-5f9d949bd namespace: kserve-ci-e2e-test resourceVersion: "25554" uid: 38699207-aacd-40b8-a8b5-e6922e51fdcc kind: Event lastTimestamp: "2026-07-02T14:47:59Z" message: 'Created pod: success-200-isvc-2963e-predictor-5f9d949bd-h2vcz' metadata: creationTimestamp: "2026-07-02T14:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:47:59Z" name: success-200-isvc-2963e-predictor-5f9d949bd.18be80666f4b29b2 namespace: kserve-ci-e2e-test resourceVersion: "25577" uid: 403494e0-748e-4b8c-80d4-b2355ea7d37f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-2963e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25551" uid: 2964eb6b-6309-45b5-ac7a-a83217f42576 kind: Event lastTimestamp: "2026-07-02T14:47:59Z" message: Scaled up replica set success-200-isvc-2963e-predictor-5f9d949bd from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:47:59Z" name: success-200-isvc-2963e-predictor.18be80666d5d26fa namespace: kserve-ci-e2e-test resourceVersion: "25558" uid: 50125b4f-65cc-416e-8415-d7dcfc79efb2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:48:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2963e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25556" uid: 80e7e628-75f0-4a1d-ba4a-6a14b7a0efe9 kind: Event lastTimestamp: "2026-07-02T14:48:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:48:44Z" name: success-200-isvc-2963e-predictor.18be8069ec326440 namespace: kserve-ci-e2e-test resourceVersion: "26361" uid: 111dcc0f-5d74-468a-add5-d0c407db18b9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:48:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2963e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25556" uid: 80e7e628-75f0-4a1d-ba4a-6a14b7a0efe9 kind: Event lastTimestamp: "2026-07-02T14:48:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:48:44Z" name: success-200-isvc-2963e-predictor.18be8069ec83f73f namespace: kserve-ci-e2e-test resourceVersion: "26362" uid: 94c9f2eb-a94e-41cb-9370-6f5801c8fe05 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:48:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2963e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25826" uid: 80e7e628-75f0-4a1d-ba4a-6a14b7a0efe9 kind: Event lastTimestamp: "2026-07-02T14:49:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:49:29Z" name: success-200-isvc-2963e-predictor.18be807468c2fc70 namespace: kserve-ci-e2e-test resourceVersion: "26828" uid: e6722160-9dc8-4204-ab10-bdecac3395a8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:48:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2963e-predictor namespace: kserve-ci-e2e-test resourceVersion: "25826" uid: 80e7e628-75f0-4a1d-ba4a-6a14b7a0efe9 kind: Event lastTimestamp: "2026-07-02T14:49:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:49:29Z" name: success-200-isvc-2963e-predictor.18be8074693565fe namespace: kserve-ci-e2e-test resourceVersion: "26829" uid: 0b3c9172-7e93-40c0-98ef-04c6caf8bf7b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:47:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2963e namespace: kserve-ci-e2e-test resourceVersion: "25533" uid: 0b77b396-d80d-4853-b5fb-5ce47396706f kind: Event lastTimestamp: "2026-07-02T14:48:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:48:57Z" name: success-200-isvc-2963e.18be80666ded8903 namespace: kserve-ci-e2e-test resourceVersion: "26492" uid: 47de2897-9a95-40ec-8186-492f3b989d3d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2963e namespace: kserve-ci-e2e-test resourceVersion: "26481" uid: 0b77b396-d80d-4853-b5fb-5ce47396706f kind: Event lastTimestamp: "2026-07-02T14:48:57Z" message: InferenceService [success-200-isvc-2963e] is Ready metadata: creationTimestamp: "2026-07-02T14:48:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:48:57Z" name: success-200-isvc-2963e.18be8073ce1615d7 namespace: kserve-ci-e2e-test resourceVersion: "26483" uid: d0b8b583-14f3-4dc6-a121-0f007cfa4b80 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:46:49.664422Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j namespace: kserve-ci-e2e-test resourceVersion: "24733" uid: 03b37e0d-c12b-4e34-bcb5-bf622dee8b92 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-484ca-predictor-6c98768cdb-w6x8j to ip-10-0-134-54.ec2.internal metadata: creationTimestamp: "2026-07-02T14:46:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:46:49Z" name: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j.18be80561349b74b namespace: kserve-ci-e2e-test resourceVersion: "24740" uid: ead67981-fb12-46e9-b649-b4d119d86298 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:49Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j namespace: kserve-ci-e2e-test resourceVersion: "24736" uid: 03b37e0d-c12b-4e34-bcb5-bf622dee8b92 kind: Event lastTimestamp: "2026-07-02T14:46:49Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-484ca-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:46:49Z" name: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j.18be805620e24f85 namespace: kserve-ci-e2e-test resourceVersion: "24789" uid: 29220c75-0820-40b5-8df7-0d6f0400631f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:50Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j namespace: kserve-ci-e2e-test resourceVersion: "24749" uid: 03b37e0d-c12b-4e34-bcb5-bf622dee8b92 kind: Event lastTimestamp: "2026-07-02T14:46:50Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:46:50Z" name: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j.18be805650c7140e namespace: kserve-ci-e2e-test resourceVersion: "24805" uid: 9c1ac652-7f17-46c7-99ad-159d07b00752 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j namespace: kserve-ci-e2e-test resourceVersion: "24736" uid: 03b37e0d-c12b-4e34-bcb5-bf622dee8b92 kind: Event lastTimestamp: "2026-07-02T14:46:50Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:46:50Z" name: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j.18be8056528c9107 namespace: kserve-ci-e2e-test resourceVersion: "24807" uid: 1367c33a-69d9-4be5-afbd-38cf6d1d4228 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j namespace: kserve-ci-e2e-test resourceVersion: "24736" uid: 03b37e0d-c12b-4e34-bcb5-bf622dee8b92 kind: Event lastTimestamp: "2026-07-02T14:46:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:46:50Z" name: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j.18be805655e9d117 namespace: kserve-ci-e2e-test resourceVersion: "24810" uid: fced1cec-202d-4013-8f94-1fee1894e6c2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j namespace: kserve-ci-e2e-test resourceVersion: "24736" uid: 03b37e0d-c12b-4e34-bcb5-bf622dee8b92 kind: Event lastTimestamp: "2026-07-02T14:46:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:46:50Z" name: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j.18be805656d8070c namespace: kserve-ci-e2e-test resourceVersion: "24812" uid: bc24c7d8-5590-409e-a5e9-11e9f9fc1285 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j namespace: kserve-ci-e2e-test resourceVersion: "24736" uid: 03b37e0d-c12b-4e34-bcb5-bf622dee8b92 kind: Event lastTimestamp: "2026-07-02T14:46:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:46:50Z" name: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j.18be805656e676fd namespace: kserve-ci-e2e-test resourceVersion: "24813" uid: 65201bf9-ac0e-4571-b4c7-51e00e5e3a59 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j namespace: kserve-ci-e2e-test resourceVersion: "24736" uid: 03b37e0d-c12b-4e34-bcb5-bf622dee8b92 kind: Event lastTimestamp: "2026-07-02T14:46:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:46:50Z" name: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j.18be80565a0a23bb namespace: kserve-ci-e2e-test resourceVersion: "24815" uid: 2613dace-88ae-44db-ad98-f7072f568648 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j namespace: kserve-ci-e2e-test resourceVersion: "24736" uid: 03b37e0d-c12b-4e34-bcb5-bf622dee8b92 kind: Event lastTimestamp: "2026-07-02T14:46:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:46:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:46:50Z" name: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j.18be80565aca43d6 namespace: kserve-ci-e2e-test resourceVersion: "24819" uid: 782b993c-c54f-4e12-b857-8dc62fdccd59 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:46:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j namespace: kserve-ci-e2e-test resourceVersion: "24736" uid: 03b37e0d-c12b-4e34-bcb5-bf622dee8b92 kind: Event lastTimestamp: "2026-07-02T14:47:27Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:47:27Z" name: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j.18be805696e8b9b3 namespace: kserve-ci-e2e-test resourceVersion: "25120" uid: 1378749c-d275-401d-bd94-a02765a930fd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j namespace: kserve-ci-e2e-test resourceVersion: "24736" uid: 03b37e0d-c12b-4e34-bcb5-bf622dee8b92 kind: Event lastTimestamp: "2026-07-02T14:47:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:47:59Z" name: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j.18be80666ff8e9f0 namespace: kserve-ci-e2e-test resourceVersion: "25581" uid: ff2714b4-efd1-4433-9c4b-cbb23c311a4b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j namespace: kserve-ci-e2e-test resourceVersion: "24736" uid: 03b37e0d-c12b-4e34-bcb5-bf622dee8b92 kind: Event lastTimestamp: "2026-07-02T14:47:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:47:59Z" name: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j.18be80666fff664c namespace: kserve-ci-e2e-test resourceVersion: "25583" uid: 4cf4f494-926a-428b-a3b7-332eb5b4e8f0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j namespace: kserve-ci-e2e-test resourceVersion: "24736" uid: 03b37e0d-c12b-4e34-bcb5-bf622dee8b92 kind: Event lastTimestamp: "2026-07-02T14:48:02Z" message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": dial tcp 10.133.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:48:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:02Z" name: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j.18be80671f27f831 namespace: kserve-ci-e2e-test resourceVersion: "25705" uid: 9b1e64a8-ac98-499b-9b0d-6b2c82d158ef reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-484ca-predictor-6c98768cdb namespace: kserve-ci-e2e-test resourceVersion: "24714" uid: 38bcfb47-6fde-458a-8519-3b997cea01e5 kind: Event lastTimestamp: "2026-07-02T14:46:49Z" message: 'Created pod: success-200-isvc-484ca-predictor-6c98768cdb-w6x8j' metadata: creationTimestamp: "2026-07-02T14:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:46:49Z" name: success-200-isvc-484ca-predictor-6c98768cdb.18be805612ceb3c1 namespace: kserve-ci-e2e-test resourceVersion: "24735" uid: 9bfd26dd-0d11-49b2-b314-e0c7ad89902e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-484ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "24707" uid: 72366988-b4ac-40f9-a1ff-c2546b9376b8 kind: Event lastTimestamp: "2026-07-02T14:46:49Z" message: Scaled up replica set success-200-isvc-484ca-predictor-6c98768cdb from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:46:49Z" name: success-200-isvc-484ca-predictor.18be80560ed4ee1f namespace: kserve-ci-e2e-test resourceVersion: "24720" uid: 0d45b709-f849-457c-bc2d-0c53695ab140 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:47:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-484ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "24713" uid: 2226aba5-c429-4363-badb-bde53ed48c04 kind: Event lastTimestamp: "2026-07-02T14:47:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:47:19Z" name: success-200-isvc-484ca-predictor.18be80598d2e5139 namespace: kserve-ci-e2e-test resourceVersion: "25054" uid: a6ea9644-5dc5-4316-a812-300f10bb454d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:47:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-484ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "24713" uid: 2226aba5-c429-4363-badb-bde53ed48c04 kind: Event lastTimestamp: "2026-07-02T14:47:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:47:19Z" name: success-200-isvc-484ca-predictor.18be80598da0ae1d namespace: kserve-ci-e2e-test resourceVersion: "25056" uid: 11fa10b2-b0d0-489a-9f2e-fbf1a1298eea reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:47:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-484ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "24942" uid: 2226aba5-c429-4363-badb-bde53ed48c04 kind: Event lastTimestamp: "2026-07-02T14:47:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:47:49Z" name: success-200-isvc-484ca-predictor.18be80608af23603 namespace: kserve-ci-e2e-test resourceVersion: "25289" uid: bb17cd66-bdb3-4e63-8d35-3b102cf8bad0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:47:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-484ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "24942" uid: 2226aba5-c429-4363-badb-bde53ed48c04 kind: Event lastTimestamp: "2026-07-02T14:47:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:47:49Z" name: success-200-isvc-484ca-predictor.18be80608b5aed5c namespace: kserve-ci-e2e-test resourceVersion: "25290" uid: 463ef92c-54ce-4f9a-ba97-09733650964b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:46:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-484ca namespace: kserve-ci-e2e-test resourceVersion: "24700" uid: 159b3bb5-66f3-4ba4-8d36-fc92740b29dc kind: Event lastTimestamp: "2026-07-02T14:47:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:47:38Z" name: success-200-isvc-484ca.18be80560ee42292 namespace: kserve-ci-e2e-test resourceVersion: "25208" uid: e0be9a41-f3e3-4800-9637-06a0931410bc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-484ca namespace: kserve-ci-e2e-test resourceVersion: "24700" uid: 159b3bb5-66f3-4ba4-8d36-fc92740b29dc kind: Event lastTimestamp: "2026-07-02T14:46:49Z" message: 'Failed to update status for InferenceService "success-200-isvc-484ca": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-484ca": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:46:49Z" name: success-200-isvc-484ca.18be80560f699c56 namespace: kserve-ci-e2e-test resourceVersion: "24726" uid: a3c5fe49-c2c2-4e2d-8c31-3f790d6da43d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-484ca namespace: kserve-ci-e2e-test resourceVersion: "24700" uid: 159b3bb5-66f3-4ba4-8d36-fc92740b29dc kind: Event lastTimestamp: "2026-07-02T14:46:49Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-484ca": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:46:49Z" name: success-200-isvc-484ca.18be80560f69f62f namespace: kserve-ci-e2e-test resourceVersion: "24729" uid: 6ae4252a-a444-472c-aff5-e8c47c4dd355 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-484ca namespace: kserve-ci-e2e-test resourceVersion: "25205" uid: 159b3bb5-66f3-4ba4-8d36-fc92740b29dc kind: Event lastTimestamp: "2026-07-02T14:47:38Z" message: InferenceService [success-200-isvc-484ca] is Ready metadata: creationTimestamp: "2026-07-02T14:47:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:47:38Z" name: success-200-isvc-484ca.18be80615612e721 namespace: kserve-ci-e2e-test resourceVersion: "25206" uid: 38c93485-699b-4f31-9a05-c7158cb3d3ea reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:37:34.714626Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-57d01-predictor-5bb7475544-nw2mc namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: 9b829695-5466-4067-a183-a1a7434cb2c8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-57d01-predictor-5bb7475544-nw2mc to ip-10-0-134-54.ec2.internal metadata: creationTimestamp: "2026-07-02T14:37:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:37:34Z" name: success-200-isvc-57d01-predictor-5bb7475544-nw2mc.18be7fd4ddb3bfe3 namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: a58fe64a-bb7f-45a2-995b-c3d65839dcc0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:34Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-57d01-predictor-5bb7475544-nw2mc namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: 9b829695-5466-4067-a183-a1a7434cb2c8 kind: Event lastTimestamp: "2026-07-02T14:37:34Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-57d01-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:34Z" name: success-200-isvc-57d01-predictor-5bb7475544-nw2mc.18be7fd4ec8e3081 namespace: kserve-ci-e2e-test resourceVersion: "20383" uid: f1603b8e-9fd6-4be4-80a8-3c43f2988fa1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:35Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-57d01-predictor-5bb7475544-nw2mc namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: 9b829695-5466-4067-a183-a1a7434cb2c8 kind: Event lastTimestamp: "2026-07-02T14:37:35Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:37:35Z" name: success-200-isvc-57d01-predictor-5bb7475544-nw2mc.18be7fd51b981ccb namespace: kserve-ci-e2e-test resourceVersion: "20410" uid: 00675bcc-58cb-45da-96a0-4c29e219ce21 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-57d01-predictor-5bb7475544-nw2mc namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: 9b829695-5466-4067-a183-a1a7434cb2c8 kind: Event lastTimestamp: "2026-07-02T14:37:35Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:35Z" name: success-200-isvc-57d01-predictor-5bb7475544-nw2mc.18be7fd51cf831c2 namespace: kserve-ci-e2e-test resourceVersion: "20412" uid: 0579bb31-fd42-4b79-940f-c3f60db712b2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-57d01-predictor-5bb7475544-nw2mc namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: 9b829695-5466-4067-a183-a1a7434cb2c8 kind: Event lastTimestamp: "2026-07-02T14:37:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:35Z" name: success-200-isvc-57d01-predictor-5bb7475544-nw2mc.18be7fd5209aa76d namespace: kserve-ci-e2e-test resourceVersion: "20413" uid: 7fa97291-3400-4aa3-a577-e195cf51dc09 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-57d01-predictor-5bb7475544-nw2mc namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: 9b829695-5466-4067-a183-a1a7434cb2c8 kind: Event lastTimestamp: "2026-07-02T14:37:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:35Z" name: success-200-isvc-57d01-predictor-5bb7475544-nw2mc.18be7fd5212a64bc namespace: kserve-ci-e2e-test resourceVersion: "20414" uid: b6c7c7fa-aa41-4286-bd03-a747bb24d80b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-57d01-predictor-5bb7475544-nw2mc namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: 9b829695-5466-4067-a183-a1a7434cb2c8 kind: Event lastTimestamp: "2026-07-02T14:37:35Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:35Z" name: success-200-isvc-57d01-predictor-5bb7475544-nw2mc.18be7fd5213412fd namespace: kserve-ci-e2e-test resourceVersion: "20415" uid: 669da32a-5389-48ff-abb6-9de3f17ed1f1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-57d01-predictor-5bb7475544-nw2mc namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: 9b829695-5466-4067-a183-a1a7434cb2c8 kind: Event lastTimestamp: "2026-07-02T14:37:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:35Z" name: success-200-isvc-57d01-predictor-5bb7475544-nw2mc.18be7fd5242d3339 namespace: kserve-ci-e2e-test resourceVersion: "20417" uid: cdce51b8-35a7-48af-b343-ec34780c9aaa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-57d01-predictor-5bb7475544-nw2mc namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: 9b829695-5466-4067-a183-a1a7434cb2c8 kind: Event lastTimestamp: "2026-07-02T14:37:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:35Z" name: success-200-isvc-57d01-predictor-5bb7475544-nw2mc.18be7fd524a2c8d8 namespace: kserve-ci-e2e-test resourceVersion: "20418" uid: 231a0f38-df01-49a4-b0c8-53787a5dbe0d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T14:37:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-57d01-predictor-5bb7475544-nw2mc namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: 9b829695-5466-4067-a183-a1a7434cb2c8 kind: Event lastTimestamp: "2026-07-02T14:46:52Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:46:52Z" name: success-200-isvc-57d01-predictor-5bb7475544-nw2mc.18be7fd554fa606d namespace: kserve-ci-e2e-test resourceVersion: "24844" uid: 2611e210-11e0-437d-ae53-7b39a5632246 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-57d01-predictor-5bb7475544-nw2mc namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: 9b829695-5466-4067-a183-a1a7434cb2c8 kind: Event lastTimestamp: "2026-07-02T14:46:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:46:49Z" name: success-200-isvc-57d01-predictor-5bb7475544-nw2mc.18be80560ff967c2 namespace: kserve-ci-e2e-test resourceVersion: "24728" uid: 4cbc2590-9166-4268-8044-67181bf05679 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-57d01-predictor-5bb7475544-nw2mc namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: 9b829695-5466-4067-a183-a1a7434cb2c8 kind: Event lastTimestamp: "2026-07-02T14:46:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:46:49Z" name: success-200-isvc-57d01-predictor-5bb7475544-nw2mc.18be80560ff9ec79 namespace: kserve-ci-e2e-test resourceVersion: "24730" uid: bf06ee81-3e20-484e-8b71-ab185dd5b9a6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:46:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-57d01-predictor-5bb7475544-nw2mc namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: 9b829695-5466-4067-a183-a1a7434cb2c8 kind: Event lastTimestamp: "2026-07-02T14:46:52Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:46:52Z" name: success-200-isvc-57d01-predictor-5bb7475544-nw2mc.18be8056c9601500 namespace: kserve-ci-e2e-test resourceVersion: "24843" uid: 68ab59cb-cb53-4c49-a2aa-d30d9d012cae reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-57d01-predictor-5bb7475544 namespace: kserve-ci-e2e-test resourceVersion: "20307" uid: 9df49b69-1d9b-4ac6-bbdd-bbc52c6b21cc kind: Event lastTimestamp: "2026-07-02T14:37:34Z" message: 'Created pod: success-200-isvc-57d01-predictor-5bb7475544-nw2mc' metadata: creationTimestamp: "2026-07-02T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:37:34Z" name: success-200-isvc-57d01-predictor-5bb7475544.18be7fd4dd39ec94 namespace: kserve-ci-e2e-test resourceVersion: "20325" uid: fbc99285-ec0d-4471-ab6c-f5e73b738ab9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-57d01-predictor namespace: kserve-ci-e2e-test resourceVersion: "20299" uid: 7561f029-fd27-4dee-9f56-872cc1625c1a kind: Event lastTimestamp: "2026-07-02T14:37:34Z" message: Scaled up replica set success-200-isvc-57d01-predictor-5bb7475544 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:37:34Z" name: success-200-isvc-57d01-predictor.18be7fd4db4b1542 namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: 3294401e-f8f0-4a74-92b8-175a1ae07240 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:37:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-57d01-predictor namespace: kserve-ci-e2e-test resourceVersion: "20305" uid: dbd1ba9c-4f26-4859-8533-24caa3c6f33f kind: Event lastTimestamp: "2026-07-02T14:38:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:04Z" name: success-200-isvc-57d01-predictor.18be7fd85a3360e2 namespace: kserve-ci-e2e-test resourceVersion: "20669" uid: 670905da-f692-40ca-b44d-232a478fca24 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:37:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-57d01-predictor namespace: kserve-ci-e2e-test resourceVersion: "20305" uid: dbd1ba9c-4f26-4859-8533-24caa3c6f33f kind: Event lastTimestamp: "2026-07-02T14:38:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:04Z" name: success-200-isvc-57d01-predictor.18be7fd85a924034 namespace: kserve-ci-e2e-test resourceVersion: "20671" uid: bb5935d2-857b-455b-9b2a-8085da486288 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:38:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-57d01-predictor namespace: kserve-ci-e2e-test resourceVersion: "20543" uid: dbd1ba9c-4f26-4859-8533-24caa3c6f33f kind: Event lastTimestamp: "2026-07-02T14:38:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:38:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:49Z" name: success-200-isvc-57d01-predictor.18be7fdf5868777c namespace: kserve-ci-e2e-test resourceVersion: "21281" uid: c11d20f2-3d25-47b7-b1f5-8599c861e91d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:38:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-57d01-predictor namespace: kserve-ci-e2e-test resourceVersion: "20543" uid: dbd1ba9c-4f26-4859-8533-24caa3c6f33f kind: Event lastTimestamp: "2026-07-02T14:38:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:38:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:49Z" name: success-200-isvc-57d01-predictor.18be7fdf58df8da2 namespace: kserve-ci-e2e-test resourceVersion: "21283" uid: 9cf8b015-9375-405b-b500-040b30da713b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:37:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-57d01 namespace: kserve-ci-e2e-test resourceVersion: "20291" uid: d1438c99-f569-4221-97c7-f8993d2df44e kind: Event lastTimestamp: "2026-07-02T14:38:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:38:22Z" name: success-200-isvc-57d01.18be7fd4db5ff09a namespace: kserve-ci-e2e-test resourceVersion: "20847" uid: b4570c23-b84e-43a4-b2f1-251fd0120b56 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-57d01 namespace: kserve-ci-e2e-test resourceVersion: "20291" uid: d1438c99-f569-4221-97c7-f8993d2df44e kind: Event lastTimestamp: "2026-07-02T14:37:34Z" message: 'Failed to update status for InferenceService "success-200-isvc-57d01": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-57d01": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:37:34Z" name: success-200-isvc-57d01.18be7fd4dc1c7fab namespace: kserve-ci-e2e-test resourceVersion: "20319" uid: 674c9e77-d690-4f9c-b422-e4f66e41e781 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-57d01 namespace: kserve-ci-e2e-test resourceVersion: "20291" uid: d1438c99-f569-4221-97c7-f8993d2df44e kind: Event lastTimestamp: "2026-07-02T14:37:34Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-57d01": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:37:34Z" name: success-200-isvc-57d01.18be7fd4dc1cd8f5 namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: 8112ef94-9e53-4161-b45b-d2436a9adee7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-57d01 namespace: kserve-ci-e2e-test resourceVersion: "20845" uid: d1438c99-f569-4221-97c7-f8993d2df44e kind: Event lastTimestamp: "2026-07-02T14:38:22Z" message: InferenceService [success-200-isvc-57d01] is Ready metadata: creationTimestamp: "2026-07-02T14:38:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:38:22Z" name: success-200-isvc-57d01.18be7fe010a5098e namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: e314f089-f4c6-43c1-9306-981d3ff6d27f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:37:20.221220Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8937c-predictor-775f4ccc46-j956h namespace: kserve-ci-e2e-test resourceVersion: "20022" uid: 130d2b67-9d70-4a56-b596-122ef94e67cf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8937c-predictor-775f4ccc46-j956h to ip-10-0-134-54.ec2.internal metadata: creationTimestamp: "2026-07-02T14:37:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:37:20Z" name: success-200-isvc-8937c-predictor-775f4ccc46-j956h.18be7fd17dd3e504 namespace: kserve-ci-e2e-test resourceVersion: "20030" uid: a8518771-c019-4afb-9284-fded2c541428 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:20Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8937c-predictor-775f4ccc46-j956h namespace: kserve-ci-e2e-test resourceVersion: "20036" uid: 130d2b67-9d70-4a56-b596-122ef94e67cf kind: Event lastTimestamp: "2026-07-02T14:37:20Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:37:20Z" name: success-200-isvc-8937c-predictor-775f4ccc46-j956h.18be7fd19979a5d8 namespace: kserve-ci-e2e-test resourceVersion: "20091" uid: 74765ca5-9aee-449c-995d-d072afe2b82b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8937c-predictor-775f4ccc46-j956h namespace: kserve-ci-e2e-test resourceVersion: "20027" uid: 130d2b67-9d70-4a56-b596-122ef94e67cf kind: Event lastTimestamp: "2026-07-02T14:37:20Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:20Z" name: success-200-isvc-8937c-predictor-775f4ccc46-j956h.18be7fd19b0d8cd0 namespace: kserve-ci-e2e-test resourceVersion: "20093" uid: 171084d0-6487-414c-bd9d-332f44fa7bfa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8937c-predictor-775f4ccc46-j956h namespace: kserve-ci-e2e-test resourceVersion: "20027" uid: 130d2b67-9d70-4a56-b596-122ef94e67cf kind: Event lastTimestamp: "2026-07-02T14:37:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:20Z" name: success-200-isvc-8937c-predictor-775f4ccc46-j956h.18be7fd19da1f6ce namespace: kserve-ci-e2e-test resourceVersion: "20095" uid: 9ea628f9-8270-4c29-8e94-6cf929e6b592 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8937c-predictor-775f4ccc46-j956h namespace: kserve-ci-e2e-test resourceVersion: "20027" uid: 130d2b67-9d70-4a56-b596-122ef94e67cf kind: Event lastTimestamp: "2026-07-02T14:37:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:20Z" name: success-200-isvc-8937c-predictor-775f4ccc46-j956h.18be7fd19e10bf1d namespace: kserve-ci-e2e-test resourceVersion: "20096" uid: 4ddb4dca-bb5a-4262-b6d3-637a825d9029 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8937c-predictor-775f4ccc46-j956h namespace: kserve-ci-e2e-test resourceVersion: "20027" uid: 130d2b67-9d70-4a56-b596-122ef94e67cf kind: Event lastTimestamp: "2026-07-02T14:37:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:20Z" name: success-200-isvc-8937c-predictor-775f4ccc46-j956h.18be7fd19e1a88bb namespace: kserve-ci-e2e-test resourceVersion: "20097" uid: e2ee0f8f-6e87-4f1b-a55a-31c321afa7dc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8937c-predictor-775f4ccc46-j956h namespace: kserve-ci-e2e-test resourceVersion: "20027" uid: 130d2b67-9d70-4a56-b596-122ef94e67cf kind: Event lastTimestamp: "2026-07-02T14:37:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:20Z" name: success-200-isvc-8937c-predictor-775f4ccc46-j956h.18be7fd1a0f0d1b1 namespace: kserve-ci-e2e-test resourceVersion: "20098" uid: de9c27ff-f44f-4257-9396-71f263bb063b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8937c-predictor-775f4ccc46-j956h namespace: kserve-ci-e2e-test resourceVersion: "20027" uid: 130d2b67-9d70-4a56-b596-122ef94e67cf kind: Event lastTimestamp: "2026-07-02T14:37:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:20Z" name: success-200-isvc-8937c-predictor-775f4ccc46-j956h.18be7fd1a1609821 namespace: kserve-ci-e2e-test resourceVersion: "20099" uid: 31112c80-2fb4-411e-9f25-384b0a674cc3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T14:37:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8937c-predictor-775f4ccc46-j956h namespace: kserve-ci-e2e-test resourceVersion: "20027" uid: 130d2b67-9d70-4a56-b596-122ef94e67cf kind: Event lastTimestamp: "2026-07-02T14:38:08Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:37:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:08Z" name: success-200-isvc-8937c-predictor-775f4ccc46-j956h.18be7fd20e189581 namespace: kserve-ci-e2e-test resourceVersion: "20710" uid: de04ac10-0c12-40dd-a64a-a110e5d230c2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8937c-predictor-775f4ccc46-j956h namespace: kserve-ci-e2e-test resourceVersion: "20027" uid: 130d2b67-9d70-4a56-b596-122ef94e67cf kind: Event lastTimestamp: "2026-07-02T14:38:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:44Z" name: success-200-isvc-8937c-predictor-775f4ccc46-j956h.18be7fe529bcd72a namespace: kserve-ci-e2e-test resourceVersion: "21179" uid: 8a1b5570-1f50-4774-b4c6-b4cdcde5336d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8937c-predictor-775f4ccc46-j956h namespace: kserve-ci-e2e-test resourceVersion: "20027" uid: 130d2b67-9d70-4a56-b596-122ef94e67cf kind: Event lastTimestamp: "2026-07-02T14:38:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:44Z" name: success-200-isvc-8937c-predictor-775f4ccc46-j956h.18be7fe529be04d6 namespace: kserve-ci-e2e-test resourceVersion: "21183" uid: 53fdc8c4-f31b-4f55-8afe-2740c71a3e54 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8937c-predictor-775f4ccc46-j956h namespace: kserve-ci-e2e-test resourceVersion: "20027" uid: 130d2b67-9d70-4a56-b596-122ef94e67cf kind: Event lastTimestamp: "2026-07-02T14:38:49Z" message: 'Readiness probe failed: Get "https://10.133.0.30:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-07-02T14:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:49Z" name: success-200-isvc-8937c-predictor-775f4ccc46-j956h.18be7fe64fe45fbc namespace: kserve-ci-e2e-test resourceVersion: "21278" uid: dca013d4-5ff3-4713-bb4b-1b43fab4af31 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8937c-predictor-775f4ccc46-j956h namespace: kserve-ci-e2e-test resourceVersion: "20027" uid: 130d2b67-9d70-4a56-b596-122ef94e67cf kind: Event lastTimestamp: "2026-07-02T14:38:49Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: i/o timeout' metadata: creationTimestamp: "2026-07-02T14:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:49Z" name: success-200-isvc-8937c-predictor-775f4ccc46-j956h.18be7fe650345b5f namespace: kserve-ci-e2e-test resourceVersion: "21279" uid: 89c6c201-66e3-41ed-96ee-1159dbdf34b9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8937c-predictor-775f4ccc46 namespace: kserve-ci-e2e-test resourceVersion: "20005" uid: 786a193f-6409-4e8a-ab96-6aee7420585d kind: Event lastTimestamp: "2026-07-02T14:37:20Z" message: 'Created pod: success-200-isvc-8937c-predictor-775f4ccc46-j956h' metadata: creationTimestamp: "2026-07-02T14:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:37:20Z" name: success-200-isvc-8937c-predictor-775f4ccc46.18be7fd17d4e7307 namespace: kserve-ci-e2e-test resourceVersion: "20024" uid: 5f4683d3-ba5d-4561-bcde-52ad887c2573 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8937c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20001" uid: e5c95d63-3115-4c91-a484-6f7f1f6dd14f kind: Event lastTimestamp: "2026-07-02T14:37:20Z" message: Scaled up replica set success-200-isvc-8937c-predictor-775f4ccc46 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:37:20Z" name: success-200-isvc-8937c-predictor.18be7fd17b74ce0e namespace: kserve-ci-e2e-test resourceVersion: "20012" uid: d21a735e-ec32-4c48-b05c-034c7d291d19 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:37:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8937c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20003" uid: 8de4fc13-b844-4c5a-a5ab-5b0c1f02c42b kind: Event lastTimestamp: "2026-07-02T14:38:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:05Z" name: success-200-isvc-8937c-predictor.18be7fd4f9d6bea3 namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: d9c72cb2-6c78-46a0-a6be-fc4c8e267e6e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:37:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8937c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20003" uid: 8de4fc13-b844-4c5a-a5ab-5b0c1f02c42b kind: Event lastTimestamp: "2026-07-02T14:38:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:05Z" name: success-200-isvc-8937c-predictor.18be7fd4fa44b30c namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: 1511aaa2-c23e-4b6c-9f74-3ebb12f22ad4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:38:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8937c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20391" uid: 8de4fc13-b844-4c5a-a5ab-5b0c1f02c42b kind: Event lastTimestamp: "2026-07-02T14:38:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:38:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:35Z" name: success-200-isvc-8937c-predictor.18be7fdf763c0aa2 namespace: kserve-ci-e2e-test resourceVersion: "21014" uid: 3ac040e6-2f9e-4b41-91ce-f0bdc32c8259 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:38:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8937c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20391" uid: 8de4fc13-b844-4c5a-a5ab-5b0c1f02c42b kind: Event lastTimestamp: "2026-07-02T14:38:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:38:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:35Z" name: success-200-isvc-8937c-predictor.18be7fdf76a9aeb1 namespace: kserve-ci-e2e-test resourceVersion: "21015" uid: ff316ed4-18cf-4d0b-9aba-1c22b07e9551 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:37:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8937c namespace: kserve-ci-e2e-test resourceVersion: "19995" uid: 87c33bf8-c3bb-4329-b9c3-15c062cda7af kind: Event lastTimestamp: "2026-07-02T14:38:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:38:18Z" name: success-200-isvc-8937c.18be7fd17b7202cf namespace: kserve-ci-e2e-test resourceVersion: "20796" uid: 0a88991e-41a8-4bd6-8b63-8be90184cf92 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8937c namespace: kserve-ci-e2e-test resourceVersion: "20791" uid: 87c33bf8-c3bb-4329-b9c3-15c062cda7af kind: Event lastTimestamp: "2026-07-02T14:38:18Z" message: InferenceService [success-200-isvc-8937c] is Ready metadata: creationTimestamp: "2026-07-02T14:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:38:18Z" name: success-200-isvc-8937c.18be7fdf1e7b5b0e namespace: kserve-ci-e2e-test resourceVersion: "20792" uid: abddfb59-6a3e-4f3d-9058-9872476adce2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:28:05.423612Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9a7ca-predictor-78f5cf9c68-56vsh namespace: kserve-ci-e2e-test resourceVersion: "15732" uid: 9f7a0abb-8b1f-4132-b57f-ba9437271dbd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-9a7ca-predictor-78f5cf9c68-56vsh to ip-10-0-134-54.ec2.internal metadata: creationTimestamp: "2026-07-02T14:28:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:28:05Z" name: success-200-isvc-9a7ca-predictor-78f5cf9c68-56vsh.18be7f50515040ae namespace: kserve-ci-e2e-test resourceVersion: "15743" uid: 2e28375f-ed7c-4c9b-b0a9-477201e1617b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:05Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9a7ca-predictor-78f5cf9c68-56vsh namespace: kserve-ci-e2e-test resourceVersion: "15750" uid: 9f7a0abb-8b1f-4132-b57f-ba9437271dbd kind: Event lastTimestamp: "2026-07-02T14:28:05Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:28:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:28:05Z" name: success-200-isvc-9a7ca-predictor-78f5cf9c68-56vsh.18be7f506b76004f namespace: kserve-ci-e2e-test resourceVersion: "15796" uid: ecd41673-3461-4420-ac21-f2e6ad32edd8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9a7ca-predictor-78f5cf9c68-56vsh namespace: kserve-ci-e2e-test resourceVersion: "15736" uid: 9f7a0abb-8b1f-4132-b57f-ba9437271dbd kind: Event lastTimestamp: "2026-07-02T14:28:05Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:28:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:05Z" name: success-200-isvc-9a7ca-predictor-78f5cf9c68-56vsh.18be7f506d08a605 namespace: kserve-ci-e2e-test resourceVersion: "15799" uid: cf35bfa2-9c97-4b44-804f-8d111490ffa9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9a7ca-predictor-78f5cf9c68-56vsh namespace: kserve-ci-e2e-test resourceVersion: "15736" uid: 9f7a0abb-8b1f-4132-b57f-ba9437271dbd kind: Event lastTimestamp: "2026-07-02T14:28:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:28:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:05Z" name: success-200-isvc-9a7ca-predictor-78f5cf9c68-56vsh.18be7f50700e1b82 namespace: kserve-ci-e2e-test resourceVersion: "15800" uid: f67030ed-f5b6-48bf-a884-ecad371df6f9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9a7ca-predictor-78f5cf9c68-56vsh namespace: kserve-ci-e2e-test resourceVersion: "15736" uid: 9f7a0abb-8b1f-4132-b57f-ba9437271dbd kind: Event lastTimestamp: "2026-07-02T14:28:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:28:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:05Z" name: success-200-isvc-9a7ca-predictor-78f5cf9c68-56vsh.18be7f5070a33c13 namespace: kserve-ci-e2e-test resourceVersion: "15801" uid: daf4ee41-f410-414c-b56d-3ec29ed986b5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9a7ca-predictor-78f5cf9c68-56vsh namespace: kserve-ci-e2e-test resourceVersion: "15736" uid: 9f7a0abb-8b1f-4132-b57f-ba9437271dbd kind: Event lastTimestamp: "2026-07-02T14:28:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:28:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:05Z" name: success-200-isvc-9a7ca-predictor-78f5cf9c68-56vsh.18be7f5070af5b0c namespace: kserve-ci-e2e-test resourceVersion: "15803" uid: 0cfd193f-bc76-4165-a302-ffe3f6a585e9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9a7ca-predictor-78f5cf9c68-56vsh namespace: kserve-ci-e2e-test resourceVersion: "15736" uid: 9f7a0abb-8b1f-4132-b57f-ba9437271dbd kind: Event lastTimestamp: "2026-07-02T14:28:06Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:28:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:06Z" name: success-200-isvc-9a7ca-predictor-78f5cf9c68-56vsh.18be7f5074a9010a namespace: kserve-ci-e2e-test resourceVersion: "15806" uid: eaf2bd96-bb3d-4583-9ab6-36b7cf806c53 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9a7ca-predictor-78f5cf9c68-56vsh namespace: kserve-ci-e2e-test resourceVersion: "15736" uid: 9f7a0abb-8b1f-4132-b57f-ba9437271dbd kind: Event lastTimestamp: "2026-07-02T14:28:06Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:28:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:06Z" name: success-200-isvc-9a7ca-predictor-78f5cf9c68-56vsh.18be7f5075283cf9 namespace: kserve-ci-e2e-test resourceVersion: "15807" uid: b9f850df-3db2-4eaf-b900-b140c3a4a4d3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T14:28:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9a7ca-predictor-78f5cf9c68-56vsh namespace: kserve-ci-e2e-test resourceVersion: "15736" uid: 9f7a0abb-8b1f-4132-b57f-ba9437271dbd kind: Event lastTimestamp: "2026-07-02T14:37:23Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:28:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:23Z" name: success-200-isvc-9a7ca-predictor-78f5cf9c68-56vsh.18be7f50caa91f9b namespace: kserve-ci-e2e-test resourceVersion: "20136" uid: 36ef44fd-cf4e-4a3e-b46f-f104fcc0e4c2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9a7ca-predictor-78f5cf9c68-56vsh namespace: kserve-ci-e2e-test resourceVersion: "15736" uid: 9f7a0abb-8b1f-4132-b57f-ba9437271dbd kind: Event lastTimestamp: "2026-07-02T14:37:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:20Z" name: success-200-isvc-9a7ca-predictor-78f5cf9c68-56vsh.18be7fd17cc1bf0a namespace: kserve-ci-e2e-test resourceVersion: "20021" uid: e39615c9-3874-4f92-8471-d2a730d30a95 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9a7ca-predictor-78f5cf9c68-56vsh namespace: kserve-ci-e2e-test resourceVersion: "15736" uid: 9f7a0abb-8b1f-4132-b57f-ba9437271dbd kind: Event lastTimestamp: "2026-07-02T14:37:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:20Z" name: success-200-isvc-9a7ca-predictor-78f5cf9c68-56vsh.18be7fd17cc2e380 namespace: kserve-ci-e2e-test resourceVersion: "20023" uid: 9a89a59a-3290-42e4-abfd-5acfef580eed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9a7ca-predictor-78f5cf9c68-56vsh namespace: kserve-ci-e2e-test resourceVersion: "15736" uid: 9f7a0abb-8b1f-4132-b57f-ba9437271dbd kind: Event lastTimestamp: "2026-07-02T14:37:23Z" message: 'Readiness probe failed: Get "https://10.133.0.26:8643/healthz": dial tcp 10.133.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:37:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:23Z" name: success-200-isvc-9a7ca-predictor-78f5cf9c68-56vsh.18be7fd23f0e0be4 namespace: kserve-ci-e2e-test resourceVersion: "20135" uid: deb0de30-b6f3-4ed7-8ad1-5d2a27e010e1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-9a7ca-predictor-78f5cf9c68 namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: 9768f942-7a67-41b6-bcbb-a184a6c196f8 kind: Event lastTimestamp: "2026-07-02T14:28:05Z" message: 'Created pod: success-200-isvc-9a7ca-predictor-78f5cf9c68-56vsh' metadata: creationTimestamp: "2026-07-02T14:28:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:05Z" name: success-200-isvc-9a7ca-predictor-78f5cf9c68.18be7f5050c872d6 namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: 33265c50-247e-4385-b023-60412dae214d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-9a7ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "15708" uid: 3f8097b4-292d-4c28-89c4-2cd69911a8ec kind: Event lastTimestamp: "2026-07-02T14:28:05Z" message: Scaled up replica set success-200-isvc-9a7ca-predictor-78f5cf9c68 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:28:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:05Z" name: success-200-isvc-9a7ca-predictor.18be7f504d84bc92 namespace: kserve-ci-e2e-test resourceVersion: "15719" uid: 0c017f67-0380-43a9-aba8-2fc81771f381 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:28:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9a7ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "15711" uid: 7c71f583-6d07-4359-9559-11cb3af7ed56 kind: Event lastTimestamp: "2026-07-02T14:28:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:28:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:35Z" name: success-200-isvc-9a7ca-predictor.18be7f53cc904cc0 namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 8e4f3a3e-066a-43d7-a301-282767357d0b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:28:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9a7ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "15711" uid: 7c71f583-6d07-4359-9559-11cb3af7ed56 kind: Event lastTimestamp: "2026-07-02T14:28:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:28:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:35Z" name: success-200-isvc-9a7ca-predictor.18be7f53cd0b60d3 namespace: kserve-ci-e2e-test resourceVersion: "16112" uid: 5697613e-edd9-4a4c-817a-83a6cb076631 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:28:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9a7ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "15993" uid: 7c71f583-6d07-4359-9559-11cb3af7ed56 kind: Event lastTimestamp: "2026-07-02T14:29:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:29:20Z" name: success-200-isvc-9a7ca-predictor.18be7f5acb46475b namespace: kserve-ci-e2e-test resourceVersion: "16507" uid: 6bda5d53-636d-4a6c-9025-5ee5f86ea3e6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:28:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9a7ca-predictor namespace: kserve-ci-e2e-test resourceVersion: "15993" uid: 7c71f583-6d07-4359-9559-11cb3af7ed56 kind: Event lastTimestamp: "2026-07-02T14:29:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:29:20Z" name: success-200-isvc-9a7ca-predictor.18be7f5acbe34b5d namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: cf3a5cb7-8300-4670-817f-cc6bcbe07ef1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:28:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9a7ca namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 4914d404-962b-4451-a020-7648f6db44e5 kind: Event lastTimestamp: "2026-07-02T14:28:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:28:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:28:53Z" name: success-200-isvc-9a7ca.18be7f504dd2c40c namespace: kserve-ci-e2e-test resourceVersion: "16272" uid: 2d5dd810-cc79-4b39-b045-18dda4c34ff8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9a7ca namespace: kserve-ci-e2e-test resourceVersion: "16270" uid: 4914d404-962b-4451-a020-7648f6db44e5 kind: Event lastTimestamp: "2026-07-02T14:28:53Z" message: InferenceService [success-200-isvc-9a7ca] is Ready metadata: creationTimestamp: "2026-07-02T14:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:28:53Z" name: success-200-isvc-9a7ca.18be7f5b867a79a6 namespace: kserve-ci-e2e-test resourceVersion: "16271" uid: 2eb69424-c7a8-402d-8c50-7f7d0926951e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:38:44.627262Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c3f32-predictor-8449d84b64-z62pq namespace: kserve-ci-e2e-test resourceVersion: "21146" uid: 9ee64839-23c4-465a-a0c5-e6daf981a63c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c3f32-predictor-8449d84b64-z62pq to ip-10-0-134-54.ec2.internal metadata: creationTimestamp: "2026-07-02T14:38:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:38:44Z" name: success-200-isvc-c3f32-predictor-8449d84b64-z62pq.18be7fe524d1e0fa namespace: kserve-ci-e2e-test resourceVersion: "21151" uid: 56056c88-0b98-47fe-bf2b-ef84776f937b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:44Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c3f32-predictor-8449d84b64-z62pq namespace: kserve-ci-e2e-test resourceVersion: "21147" uid: 9ee64839-23c4-465a-a0c5-e6daf981a63c kind: Event lastTimestamp: "2026-07-02T14:38:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-c3f32-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:44Z" name: success-200-isvc-c3f32-predictor-8449d84b64-z62pq.18be7fe530051a30 namespace: kserve-ci-e2e-test resourceVersion: "21199" uid: 97ece291-feae-45b4-b849-85811cf6f078 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:45Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c3f32-predictor-8449d84b64-z62pq namespace: kserve-ci-e2e-test resourceVersion: "21155" uid: 9ee64839-23c4-465a-a0c5-e6daf981a63c kind: Event lastTimestamp: "2026-07-02T14:38:45Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:38:45Z" name: success-200-isvc-c3f32-predictor-8449d84b64-z62pq.18be7fe56240a3e3 namespace: kserve-ci-e2e-test resourceVersion: "21218" uid: 032d89cf-8267-4b2d-94de-04455ea136e1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3f32-predictor-8449d84b64-z62pq namespace: kserve-ci-e2e-test resourceVersion: "21147" uid: 9ee64839-23c4-465a-a0c5-e6daf981a63c kind: Event lastTimestamp: "2026-07-02T14:38:45Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:45Z" name: success-200-isvc-c3f32-predictor-8449d84b64-z62pq.18be7fe563e26a08 namespace: kserve-ci-e2e-test resourceVersion: "21220" uid: 54e1c3e1-6f91-489e-8a6b-7a852f027d68 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3f32-predictor-8449d84b64-z62pq namespace: kserve-ci-e2e-test resourceVersion: "21147" uid: 9ee64839-23c4-465a-a0c5-e6daf981a63c kind: Event lastTimestamp: "2026-07-02T14:38:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:45Z" name: success-200-isvc-c3f32-predictor-8449d84b64-z62pq.18be7fe567498490 namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: bcabf655-f8a2-49ef-9041-6e1e94ee153b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3f32-predictor-8449d84b64-z62pq namespace: kserve-ci-e2e-test resourceVersion: "21147" uid: 9ee64839-23c4-465a-a0c5-e6daf981a63c kind: Event lastTimestamp: "2026-07-02T14:38:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:45Z" name: success-200-isvc-c3f32-predictor-8449d84b64-z62pq.18be7fe567cf87eb namespace: kserve-ci-e2e-test resourceVersion: "21222" uid: c06ba722-3f94-4fee-825a-e402747424cc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3f32-predictor-8449d84b64-z62pq namespace: kserve-ci-e2e-test resourceVersion: "21147" uid: 9ee64839-23c4-465a-a0c5-e6daf981a63c kind: Event lastTimestamp: "2026-07-02T14:38:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:45Z" name: success-200-isvc-c3f32-predictor-8449d84b64-z62pq.18be7fe567da94fb namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 0fea9a8c-efc7-432e-92d3-8b07bac29b05 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3f32-predictor-8449d84b64-z62pq namespace: kserve-ci-e2e-test resourceVersion: "21147" uid: 9ee64839-23c4-465a-a0c5-e6daf981a63c kind: Event lastTimestamp: "2026-07-02T14:38:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:45Z" name: success-200-isvc-c3f32-predictor-8449d84b64-z62pq.18be7fe56ba91b58 namespace: kserve-ci-e2e-test resourceVersion: "21227" uid: a910332a-50b4-4b0c-bf05-29030fbcea12 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3f32-predictor-8449d84b64-z62pq namespace: kserve-ci-e2e-test resourceVersion: "21147" uid: 9ee64839-23c4-465a-a0c5-e6daf981a63c kind: Event lastTimestamp: "2026-07-02T14:38:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:38:45Z" name: success-200-isvc-c3f32-predictor-8449d84b64-z62pq.18be7fe56c1d0748 namespace: kserve-ci-e2e-test resourceVersion: "21228" uid: a5f453f0-4b90-491f-ab0f-578b6bbd89c0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T14:38:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3f32-predictor-8449d84b64-z62pq namespace: kserve-ci-e2e-test resourceVersion: "21147" uid: 9ee64839-23c4-465a-a0c5-e6daf981a63c kind: Event lastTimestamp: "2026-07-02T14:48:02Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:02Z" name: success-200-isvc-c3f32-predictor-8449d84b64-z62pq.18be7fe577369bb6 namespace: kserve-ci-e2e-test resourceVersion: "25692" uid: 481d38bb-a1ed-408f-9b1a-8a1f19bcac49 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3f32-predictor-8449d84b64-z62pq namespace: kserve-ci-e2e-test resourceVersion: "21147" uid: 9ee64839-23c4-465a-a0c5-e6daf981a63c kind: Event lastTimestamp: "2026-07-02T14:47:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:47:59Z" name: success-200-isvc-c3f32-predictor-8449d84b64-z62pq.18be80664fa2855f namespace: kserve-ci-e2e-test resourceVersion: "25463" uid: cd19393e-d6e0-4d95-8a55-57f14f2e736c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3f32-predictor-8449d84b64-z62pq namespace: kserve-ci-e2e-test resourceVersion: "21147" uid: 9ee64839-23c4-465a-a0c5-e6daf981a63c kind: Event lastTimestamp: "2026-07-02T14:47:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:47:59Z" name: success-200-isvc-c3f32-predictor-8449d84b64-z62pq.18be80664fa2e6ec namespace: kserve-ci-e2e-test resourceVersion: "25469" uid: 47e8d11a-a103-401b-a7a6-8990c5567d47 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3f32-predictor-8449d84b64-z62pq namespace: kserve-ci-e2e-test resourceVersion: "21147" uid: 9ee64839-23c4-465a-a0c5-e6daf981a63c kind: Event lastTimestamp: "2026-07-02T14:48:02Z" message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": dial tcp 10.133.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:48:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:02Z" name: success-200-isvc-c3f32-predictor-8449d84b64-z62pq.18be8066ebbc63ee namespace: kserve-ci-e2e-test resourceVersion: "25691" uid: aea75025-26c7-4d9e-b87e-ca9dc83c471f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c3f32-predictor-8449d84b64 namespace: kserve-ci-e2e-test resourceVersion: "21136" uid: 22b33f41-1c49-46bd-b9da-aa32a4f9ba18 kind: Event lastTimestamp: "2026-07-02T14:38:44Z" message: 'Created pod: success-200-isvc-c3f32-predictor-8449d84b64-z62pq' metadata: creationTimestamp: "2026-07-02T14:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:44Z" name: success-200-isvc-c3f32-predictor-8449d84b64.18be7fe52453fd75 namespace: kserve-ci-e2e-test resourceVersion: "21148" uid: 0d725be5-cfc8-48a2-946d-544bcddbef19 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c3f32-predictor namespace: kserve-ci-e2e-test resourceVersion: "21133" uid: 91afc9f6-5167-4cb9-b744-34ace1dbd083 kind: Event lastTimestamp: "2026-07-02T14:38:44Z" message: Scaled up replica set success-200-isvc-c3f32-predictor-8449d84b64 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:38:44Z" name: success-200-isvc-c3f32-predictor.18be7fe522507af1 namespace: kserve-ci-e2e-test resourceVersion: "21138" uid: 5d93e6cf-278e-4556-8fd5-4ea77c75ed1a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:38:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3f32-predictor namespace: kserve-ci-e2e-test resourceVersion: "21135" uid: 65c39c26-9a5b-40b4-9268-287b63192ad1 kind: Event lastTimestamp: "2026-07-02T14:39:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:38:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:39:29Z" name: success-200-isvc-c3f32-predictor.18be7fe8a0a31c6e namespace: kserve-ci-e2e-test resourceVersion: "21593" uid: e83b8c79-31f4-496d-9f98-8a6e7369c1c3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:38:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3f32-predictor namespace: kserve-ci-e2e-test resourceVersion: "21135" uid: 65c39c26-9a5b-40b4-9268-287b63192ad1 kind: Event lastTimestamp: "2026-07-02T14:39:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:38:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:39:29Z" name: success-200-isvc-c3f32-predictor.18be7fe8a1109a16 namespace: kserve-ci-e2e-test resourceVersion: "21594" uid: ef00dfb4-3a79-49a2-af9e-1d265b998374 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:39:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3f32-predictor namespace: kserve-ci-e2e-test resourceVersion: "21362" uid: 65c39c26-9a5b-40b4-9268-287b63192ad1 kind: Event lastTimestamp: "2026-07-02T14:39:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:39:59Z" name: success-200-isvc-c3f32-predictor.18be7ff31dabb019 namespace: kserve-ci-e2e-test resourceVersion: "21861" uid: bf9c2e56-1590-4cab-ba5a-082f752ecf19 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:39:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3f32-predictor namespace: kserve-ci-e2e-test resourceVersion: "21362" uid: 65c39c26-9a5b-40b4-9268-287b63192ad1 kind: Event lastTimestamp: "2026-07-02T14:39:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:39:59Z" name: success-200-isvc-c3f32-predictor.18be7ff31e17735d namespace: kserve-ci-e2e-test resourceVersion: "21863" uid: cb198896-b9d5-49b1-a59f-625a7bcfda2c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:38:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3f32 namespace: kserve-ci-e2e-test resourceVersion: "21114" uid: c486b0dc-c9a8-4633-9090-346ae2501f2c kind: Event lastTimestamp: "2026-07-02T14:39:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:39:32Z" name: success-200-isvc-c3f32.18be7fe52257b43f namespace: kserve-ci-e2e-test resourceVersion: "21620" uid: 2a55e999-323c-4afb-8a87-3a84f52bb17e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3f32 namespace: kserve-ci-e2e-test resourceVersion: "21114" uid: c486b0dc-c9a8-4633-9090-346ae2501f2c kind: Event lastTimestamp: "2026-07-02T14:38:44Z" message: 'Failed to update status for InferenceService "success-200-isvc-c3f32": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c3f32": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:38:44Z" name: success-200-isvc-c3f32.18be7fe522da648b namespace: kserve-ci-e2e-test resourceVersion: "21143" uid: fd27893a-a0c9-4878-b7bc-8fe057fe4bff reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:38:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3f32 namespace: kserve-ci-e2e-test resourceVersion: "21114" uid: c486b0dc-c9a8-4633-9090-346ae2501f2c kind: Event lastTimestamp: "2026-07-02T14:38:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c3f32": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:38:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:38:44Z" name: success-200-isvc-c3f32.18be7fe522daa308 namespace: kserve-ci-e2e-test resourceVersion: "21144" uid: 890a599d-c780-4b24-895b-6d3dfc0dbfc6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:39:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3f32 namespace: kserve-ci-e2e-test resourceVersion: "21617" uid: c486b0dc-c9a8-4633-9090-346ae2501f2c kind: Event lastTimestamp: "2026-07-02T14:39:32Z" message: InferenceService [success-200-isvc-c3f32] is Ready metadata: creationTimestamp: "2026-07-02T14:39:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:39:32Z" name: success-200-isvc-c3f32.18be7ff032d43052 namespace: kserve-ci-e2e-test resourceVersion: "21619" uid: e07b8b5a-1f6e-4b1c-aaa2-3093bb92210d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:47:59.397131Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cb06c-predictor-bd6494ccd-g2kfz namespace: kserve-ci-e2e-test resourceVersion: "25458" uid: 5e99f649-b46f-4221-a022-a860fc8d998a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-cb06c-predictor-bd6494ccd-g2kfz to ip-10-0-134-54.ec2.internal metadata: creationTimestamp: "2026-07-02T14:47:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:47:59Z" name: success-200-isvc-cb06c-predictor-bd6494ccd-g2kfz.18be80664fae4bad namespace: kserve-ci-e2e-test resourceVersion: "25464" uid: 5bd9d54e-2e20-44e4-8237-e9aa43fdfb8c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:59Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cb06c-predictor-bd6494ccd-g2kfz namespace: kserve-ci-e2e-test resourceVersion: "25461" uid: 5e99f649-b46f-4221-a022-a860fc8d998a kind: Event lastTimestamp: "2026-07-02T14:47:59Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-cb06c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:47:59Z" name: success-200-isvc-cb06c-predictor-bd6494ccd-g2kfz.18be80665e61d106 namespace: kserve-ci-e2e-test resourceVersion: "25507" uid: 724da51d-6ca2-4ae8-862a-7427e5fb251f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cb06c-predictor-bd6494ccd-g2kfz namespace: kserve-ci-e2e-test resourceVersion: "25473" uid: 5e99f649-b46f-4221-a022-a860fc8d998a kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:48:00Z" name: success-200-isvc-cb06c-predictor-bd6494ccd-g2kfz.18be80668f77f944 namespace: kserve-ci-e2e-test resourceVersion: "25641" uid: a34f4a9d-aa79-4864-89e0-084ce7dfe0d2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cb06c-predictor-bd6494ccd-g2kfz namespace: kserve-ci-e2e-test resourceVersion: "25461" uid: 5e99f649-b46f-4221-a022-a860fc8d998a kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:00Z" name: success-200-isvc-cb06c-predictor-bd6494ccd-g2kfz.18be8066917a8df0 namespace: kserve-ci-e2e-test resourceVersion: "25644" uid: 62ef1fe3-aa8b-4ae1-9536-e2ca454ad4a6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cb06c-predictor-bd6494ccd-g2kfz namespace: kserve-ci-e2e-test resourceVersion: "25461" uid: 5e99f649-b46f-4221-a022-a860fc8d998a kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:00Z" name: success-200-isvc-cb06c-predictor-bd6494ccd-g2kfz.18be8066953e73d4 namespace: kserve-ci-e2e-test resourceVersion: "25649" uid: 564cabc0-3c90-48aa-a86c-ebd49065837d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cb06c-predictor-bd6494ccd-g2kfz namespace: kserve-ci-e2e-test resourceVersion: "25461" uid: 5e99f649-b46f-4221-a022-a860fc8d998a kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:00Z" name: success-200-isvc-cb06c-predictor-bd6494ccd-g2kfz.18be806695d1684c namespace: kserve-ci-e2e-test resourceVersion: "25650" uid: 2ed10db7-2bc0-46b0-a830-e29644d1ff57 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cb06c-predictor-bd6494ccd-g2kfz namespace: kserve-ci-e2e-test resourceVersion: "25461" uid: 5e99f649-b46f-4221-a022-a860fc8d998a kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:00Z" name: success-200-isvc-cb06c-predictor-bd6494ccd-g2kfz.18be806695f8ea94 namespace: kserve-ci-e2e-test resourceVersion: "25652" uid: 0951ab36-d3eb-40b8-ab82-2b982a8c5c6d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cb06c-predictor-bd6494ccd-g2kfz namespace: kserve-ci-e2e-test resourceVersion: "25461" uid: 5e99f649-b46f-4221-a022-a860fc8d998a kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:00Z" name: success-200-isvc-cb06c-predictor-bd6494ccd-g2kfz.18be80669af99844 namespace: kserve-ci-e2e-test resourceVersion: "25656" uid: 3ba65513-6b4c-4db3-abc2-d3521ee52164 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cb06c-predictor-bd6494ccd-g2kfz namespace: kserve-ci-e2e-test resourceVersion: "25461" uid: 5e99f649-b46f-4221-a022-a860fc8d998a kind: Event lastTimestamp: "2026-07-02T14:48:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:00Z" name: success-200-isvc-cb06c-predictor-bd6494ccd-g2kfz.18be80669b73b923 namespace: kserve-ci-e2e-test resourceVersion: "25657" uid: 1403556c-2aff-4634-8dec-b8e82b5ca8dd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T14:48:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cb06c-predictor-bd6494ccd-g2kfz namespace: kserve-ci-e2e-test resourceVersion: "25461" uid: 5e99f649-b46f-4221-a022-a860fc8d998a kind: Event lastTimestamp: "2026-07-02T14:48:47Z" message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:48:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:48:47Z" name: success-200-isvc-cb06c-predictor-bd6494ccd-g2kfz.18be8066b994c8cb namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: 8047769a-026b-43cc-ace7-c10281130783 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T15:05:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cb06c-predictor-bd6494ccd-g2kfz namespace: kserve-ci-e2e-test resourceVersion: "25461" uid: 5e99f649-b46f-4221-a022-a860fc8d998a kind: Event lastTimestamp: "2026-07-02T15:05:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T15:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T15:05:28Z" name: success-200-isvc-cb06c-predictor-bd6494ccd-g2kfz.18be815ab0c63ced namespace: kserve-ci-e2e-test resourceVersion: "33509" uid: 77df6ade-5237-4499-9e09-4d711d9fbce4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T15:05:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cb06c-predictor-bd6494ccd-g2kfz namespace: kserve-ci-e2e-test resourceVersion: "25461" uid: 5e99f649-b46f-4221-a022-a860fc8d998a kind: Event lastTimestamp: "2026-07-02T15:05:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T15:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T15:05:29Z" name: success-200-isvc-cb06c-predictor-bd6494ccd-g2kfz.18be815ab0c6bf04 namespace: kserve-ci-e2e-test resourceVersion: "33510" uid: 4ecf99b2-63c4-4686-b4c8-e61a1baf9c33 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-cb06c-predictor-bd6494ccd namespace: kserve-ci-e2e-test resourceVersion: "25442" uid: 90a77555-6eb4-46f0-b2dc-5d5fbb79d322 kind: Event lastTimestamp: "2026-07-02T14:47:59Z" message: 'Created pod: success-200-isvc-cb06c-predictor-bd6494ccd-g2kfz' metadata: creationTimestamp: "2026-07-02T14:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:47:59Z" name: success-200-isvc-cb06c-predictor-bd6494ccd.18be80664f3d573b namespace: kserve-ci-e2e-test resourceVersion: "25460" uid: b97e1dc8-1b86-424f-811c-cde7374890e9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:47:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-cb06c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25438" uid: b7a8bbcc-0ea5-47c3-bf27-8aece7394c3e kind: Event lastTimestamp: "2026-07-02T14:47:59Z" message: Scaled up replica set success-200-isvc-cb06c-predictor-bd6494ccd from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:47:59Z" name: success-200-isvc-cb06c-predictor.18be80664d7b540a namespace: kserve-ci-e2e-test resourceVersion: "25448" uid: d7f71881-5a56-4c82-a6cd-26a965ba8ab0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:48:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cb06c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25441" uid: 0de6bc08-959d-4dd9-a7fc-3915cba3b964 kind: Event lastTimestamp: "2026-07-02T14:48:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:48:44Z" name: success-200-isvc-cb06c-predictor.18be8069cc49eac1 namespace: kserve-ci-e2e-test resourceVersion: "26352" uid: b97656fd-8ec0-4e03-81dc-8a5979cf84db reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:48:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cb06c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25441" uid: 0de6bc08-959d-4dd9-a7fc-3915cba3b964 kind: Event lastTimestamp: "2026-07-02T14:48:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:48:44Z" name: success-200-isvc-cb06c-predictor.18be8069ccaa68a6 namespace: kserve-ci-e2e-test resourceVersion: "26353" uid: 7562ed65-7489-48f0-bf70-657cc333a738 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:48:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cb06c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25815" uid: 0de6bc08-959d-4dd9-a7fc-3915cba3b964 kind: Event lastTimestamp: "2026-07-02T14:49:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:49:14Z" name: success-200-isvc-cb06c-predictor.18be807449580592 namespace: kserve-ci-e2e-test resourceVersion: "26694" uid: 7b6cd91c-8b5a-4a2e-9819-ddedf2abf1e9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:48:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cb06c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25815" uid: 0de6bc08-959d-4dd9-a7fc-3915cba3b964 kind: Event lastTimestamp: "2026-07-02T14:49:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:48:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:49:14Z" name: success-200-isvc-cb06c-predictor.18be807449c2a382 namespace: kserve-ci-e2e-test resourceVersion: "26695" uid: 61f8f5d3-fcac-4a2f-87b8-1035c65e19e2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:47:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cb06c namespace: kserve-ci-e2e-test resourceVersion: "25418" uid: 823f13d5-7786-4701-b7e8-ebc19b3ec93a kind: Event lastTimestamp: "2026-07-02T14:48:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:47:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:48:57Z" name: success-200-isvc-cb06c.18be80664d899d9d namespace: kserve-ci-e2e-test resourceVersion: "26491" uid: fa459452-7aa0-4925-94d4-af2fe0baf261 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:48:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cb06c namespace: kserve-ci-e2e-test resourceVersion: "26477" uid: 823f13d5-7786-4701-b7e8-ebc19b3ec93a kind: Event lastTimestamp: "2026-07-02T14:48:57Z" message: InferenceService [success-200-isvc-cb06c] is Ready metadata: creationTimestamp: "2026-07-02T14:48:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:48:57Z" name: success-200-isvc-cb06c.18be8073cad7bcd8 namespace: kserve-ci-e2e-test resourceVersion: "26479" uid: 9837bb88-b301-4602-95a6-420b8f17e806 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:25:35.347039Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst namespace: kserve-ci-e2e-test resourceVersion: "13970" uid: f35882d3-cc5b-4b91-9516-aefeb824e516 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-df2c1-predictor-54bbb9f954-cbnst to ip-10-0-134-54.ec2.internal metadata: creationTimestamp: "2026-07-02T14:25:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:25:35Z" name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst.18be7f2d600d7f54 namespace: kserve-ci-e2e-test resourceVersion: "13975" uid: 9bc8a8aa-2600-4d4a-9876-05a192568917 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:35Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst namespace: kserve-ci-e2e-test resourceVersion: "13979" uid: f35882d3-cc5b-4b91-9516-aefeb824e516 kind: Event lastTimestamp: "2026-07-02T14:25:35Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:25:35Z" name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst.18be7f2d7b8456ee namespace: kserve-ci-e2e-test resourceVersion: "14055" uid: 506913dd-e6a9-4d44-9e43-5e760805294e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst namespace: kserve-ci-e2e-test resourceVersion: "13973" uid: f35882d3-cc5b-4b91-9516-aefeb824e516 kind: Event lastTimestamp: "2026-07-02T14:25:35Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1697" metadata: creationTimestamp: "2026-07-02T14:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:35Z" name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst.18be7f2d7d14044e namespace: kserve-ci-e2e-test resourceVersion: "14061" uid: 95fe1b51-576f-4a67-92b4-42c789a1e83b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst namespace: kserve-ci-e2e-test resourceVersion: "13973" uid: f35882d3-cc5b-4b91-9516-aefeb824e516 kind: Event lastTimestamp: "2026-07-02T14:25:51Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1697" in 16.093s (16.093s including waiting). Image size: 1335697024 bytes.' metadata: creationTimestamp: "2026-07-02T14:25:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:51Z" name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst.18be7f313c56b1d7 namespace: kserve-ci-e2e-test resourceVersion: "14256" uid: 86b37d7e-df80-46b7-b91b-aeab77397c2e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst namespace: kserve-ci-e2e-test resourceVersion: "13973" uid: f35882d3-cc5b-4b91-9516-aefeb824e516 kind: Event lastTimestamp: "2026-07-02T14:25:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:25:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:51Z" name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst.18be7f31406a6a90 namespace: kserve-ci-e2e-test resourceVersion: "14258" uid: 8e72e460-1b7f-4028-b9eb-5a8fcdc12375 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst namespace: kserve-ci-e2e-test resourceVersion: "13973" uid: f35882d3-cc5b-4b91-9516-aefeb824e516 kind: Event lastTimestamp: "2026-07-02T14:25:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:25:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:52Z" name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst.18be7f3140ee1d79 namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: bf5e6ea2-9491-46fb-ac60-998776525ef8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst namespace: kserve-ci-e2e-test resourceVersion: "13973" uid: f35882d3-cc5b-4b91-9516-aefeb824e516 kind: Event lastTimestamp: "2026-07-02T14:25:52Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T14:25:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:52Z" name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst.18be7f3140fa5f81 namespace: kserve-ci-e2e-test resourceVersion: "14261" uid: 6fd72cec-6c98-417d-be01-4a4eaae1e433 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst namespace: kserve-ci-e2e-test resourceVersion: "13973" uid: f35882d3-cc5b-4b91-9516-aefeb824e516 kind: Event lastTimestamp: "2026-07-02T14:25:54Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.691s (2.691s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T14:25:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:54Z" name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst.18be7f31e1642813 namespace: kserve-ci-e2e-test resourceVersion: "14282" uid: 4bc01940-6d7f-4669-bf4c-031da8ae94ec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst namespace: kserve-ci-e2e-test resourceVersion: "13973" uid: f35882d3-cc5b-4b91-9516-aefeb824e516 kind: Event lastTimestamp: "2026-07-02T14:25:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:25:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:54Z" name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst.18be7f31e702d88d namespace: kserve-ci-e2e-test resourceVersion: "14284" uid: 6f6efa0a-9f60-4f25-89ea-f2b20fb1dc7d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst namespace: kserve-ci-e2e-test resourceVersion: "13973" uid: f35882d3-cc5b-4b91-9516-aefeb824e516 kind: Event lastTimestamp: "2026-07-02T14:25:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:25:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:54Z" name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst.18be7f31e78a4341 namespace: kserve-ci-e2e-test resourceVersion: "14286" uid: 76af447f-08ad-48fe-b118-a718b6f08bea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T14:25:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst namespace: kserve-ci-e2e-test resourceVersion: "13973" uid: f35882d3-cc5b-4b91-9516-aefeb824e516 kind: Event lastTimestamp: "2026-07-02T14:27:10Z" message: 'Readiness probe failed: dial tcp 10.133.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:25:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:10Z" name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst.18be7f31ef397002 namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: d485242e-c0fc-4bc8-86fb-ee0b0463e95a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst namespace: kserve-ci-e2e-test resourceVersion: "13973" uid: f35882d3-cc5b-4b91-9516-aefeb824e516 kind: Event lastTimestamp: "2026-07-02T14:27:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:09Z" name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst.18be7f434c338259 namespace: kserve-ci-e2e-test resourceVersion: "15062" uid: 29ade3d3-f9ee-489a-b2ae-c2ee5f865e12 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst namespace: kserve-ci-e2e-test resourceVersion: "13973" uid: f35882d3-cc5b-4b91-9516-aefeb824e516 kind: Event lastTimestamp: "2026-07-02T14:27:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:09Z" name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst.18be7f434c3409f3 namespace: kserve-ci-e2e-test resourceVersion: "15065" uid: 8ac21cfa-366a-4614-8c31-bdea08d48274 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst namespace: kserve-ci-e2e-test resourceVersion: "13973" uid: f35882d3-cc5b-4b91-9516-aefeb824e516 kind: Event lastTimestamp: "2026-07-02T14:27:10Z" message: 'Readiness probe failed: Get "https://10.133.0.22:8643/healthz": dial tcp 10.133.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:27:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:10Z" name: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst.18be7f43a1738ab5 namespace: kserve-ci-e2e-test resourceVersion: "15140" uid: ce9aaf00-5de2-4463-8ed9-941492914a82 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-df2c1-predictor-54bbb9f954 namespace: kserve-ci-e2e-test resourceVersion: "13963" uid: 0c614719-00f0-4497-9e76-3255781a57cc kind: Event lastTimestamp: "2026-07-02T14:25:35Z" message: 'Created pod: success-200-isvc-df2c1-predictor-54bbb9f954-cbnst' metadata: creationTimestamp: "2026-07-02T14:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:25:35Z" name: success-200-isvc-df2c1-predictor-54bbb9f954.18be7f2d5f891283 namespace: kserve-ci-e2e-test resourceVersion: "13972" uid: aec7f6ee-49a3-4357-b406-5b114a890bc4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-df2c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13960" uid: a741559b-f109-4056-8a35-37025bb2d539 kind: Event lastTimestamp: "2026-07-02T14:25:35Z" message: Scaled up replica set success-200-isvc-df2c1-predictor-54bbb9f954 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:25:35Z" name: success-200-isvc-df2c1-predictor.18be7f2d5cc20d6f namespace: kserve-ci-e2e-test resourceVersion: "13964" uid: 724e3a43-53f1-4977-a7b4-4227b0cb6bf2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:25:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-df2c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13962" uid: 9c142b93-fc69-48c9-b10b-c6c27d64936d kind: Event lastTimestamp: "2026-07-02T14:26:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:25:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:26:20Z" name: success-200-isvc-df2c1-predictor.18be7f30db2d90b8 namespace: kserve-ci-e2e-test resourceVersion: "14531" uid: 751d3014-03ac-4693-b31b-e660407f01ae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:25:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-df2c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13962" uid: 9c142b93-fc69-48c9-b10b-c6c27d64936d kind: Event lastTimestamp: "2026-07-02T14:26:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:25:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:26:20Z" name: success-200-isvc-df2c1-predictor.18be7f30db97fed6 namespace: kserve-ci-e2e-test resourceVersion: "14532" uid: 067a09af-0926-4dc6-8ee8-e9a0de2e76a1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:26:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-df2c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: 9c142b93-fc69-48c9-b10b-c6c27d64936d kind: Event lastTimestamp: "2026-07-02T14:27:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:26:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:05Z" name: success-200-isvc-df2c1-predictor.18be7f3b58103b97 namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: e25587da-f2de-44d2-b532-7303246a4fc3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:26:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-df2c1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: 9c142b93-fc69-48c9-b10b-c6c27d64936d kind: Event lastTimestamp: "2026-07-02T14:27:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:26:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:27:05Z" name: success-200-isvc-df2c1-predictor.18be7f3b589411f8 namespace: kserve-ci-e2e-test resourceVersion: "14945" uid: 3ac45a18-b004-4160-843d-fc1c5d4e2ba4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:25:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-df2c1 namespace: kserve-ci-e2e-test resourceVersion: "13944" uid: de79a077-d81f-4975-9367-2a3978976f78 kind: Event lastTimestamp: "2026-07-02T14:26:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:25:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:26:41Z" name: success-200-isvc-df2c1.18be7f2d5ccaeac9 namespace: kserve-ci-e2e-test resourceVersion: "14718" uid: e22ff139-654b-4b96-a548-0fa549f1e7cf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-df2c1 namespace: kserve-ci-e2e-test resourceVersion: "14714" uid: de79a077-d81f-4975-9367-2a3978976f78 kind: Event lastTimestamp: "2026-07-02T14:26:41Z" message: InferenceService [success-200-isvc-df2c1] is Ready metadata: creationTimestamp: "2026-07-02T14:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:26:41Z" name: success-200-isvc-df2c1.18be7f3cae26fc4e namespace: kserve-ci-e2e-test resourceVersion: "14716" uid: b04fcc57-cf39-425d-a33c-1d30150fa7d1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:36:24.469025Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fbc8b-predictor-5f496dcc84-gpcr7 namespace: kserve-ci-e2e-test resourceVersion: "19446" uid: 3a0e9c74-f5b3-407c-a5d5-15377a551448 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-fbc8b-predictor-5f496dcc84-gpcr7 to ip-10-0-134-54.ec2.internal metadata: creationTimestamp: "2026-07-02T14:36:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:36:24Z" name: success-200-isvc-fbc8b-predictor-5f496dcc84-gpcr7.18be7fc482bcee58 namespace: kserve-ci-e2e-test resourceVersion: "19450" uid: db22fa64-7c2b-42fe-ba96-145bc6b95e67 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:24Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fbc8b-predictor-5f496dcc84-gpcr7 namespace: kserve-ci-e2e-test resourceVersion: "19456" uid: 3a0e9c74-f5b3-407c-a5d5-15377a551448 kind: Event lastTimestamp: "2026-07-02T14:36:24Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:36:24Z" name: success-200-isvc-fbc8b-predictor-5f496dcc84-gpcr7.18be7fc49ce542c4 namespace: kserve-ci-e2e-test resourceVersion: "19511" uid: f5d6b4c2-f029-4ebb-8272-09388c564202 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fbc8b-predictor-5f496dcc84-gpcr7 namespace: kserve-ci-e2e-test resourceVersion: "19448" uid: 3a0e9c74-f5b3-407c-a5d5-15377a551448 kind: Event lastTimestamp: "2026-07-02T14:36:24Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1697" already present on machine metadata: creationTimestamp: "2026-07-02T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:24Z" name: success-200-isvc-fbc8b-predictor-5f496dcc84-gpcr7.18be7fc49e458be9 namespace: kserve-ci-e2e-test resourceVersion: "19513" uid: 64039a68-0ef2-4fbd-b953-51896bb59728 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fbc8b-predictor-5f496dcc84-gpcr7 namespace: kserve-ci-e2e-test resourceVersion: "19448" uid: 3a0e9c74-f5b3-407c-a5d5-15377a551448 kind: Event lastTimestamp: "2026-07-02T14:36:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:24Z" name: success-200-isvc-fbc8b-predictor-5f496dcc84-gpcr7.18be7fc4a119dd15 namespace: kserve-ci-e2e-test resourceVersion: "19514" uid: d7ab3daf-8345-4e2e-95d9-a289eb92d3ef reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fbc8b-predictor-5f496dcc84-gpcr7 namespace: kserve-ci-e2e-test resourceVersion: "19448" uid: 3a0e9c74-f5b3-407c-a5d5-15377a551448 kind: Event lastTimestamp: "2026-07-02T14:36:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:24Z" name: success-200-isvc-fbc8b-predictor-5f496dcc84-gpcr7.18be7fc4a1c2074c namespace: kserve-ci-e2e-test resourceVersion: "19515" uid: 7a805f7e-9614-4c22-8d8e-7e55f2a40b17 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fbc8b-predictor-5f496dcc84-gpcr7 namespace: kserve-ci-e2e-test resourceVersion: "19448" uid: 3a0e9c74-f5b3-407c-a5d5-15377a551448 kind: Event lastTimestamp: "2026-07-02T14:36:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:24Z" name: success-200-isvc-fbc8b-predictor-5f496dcc84-gpcr7.18be7fc4a1cce3c9 namespace: kserve-ci-e2e-test resourceVersion: "19516" uid: 0f29242c-7a2b-40a4-bcb2-bde62f31b644 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fbc8b-predictor-5f496dcc84-gpcr7 namespace: kserve-ci-e2e-test resourceVersion: "19448" uid: 3a0e9c74-f5b3-407c-a5d5-15377a551448 kind: Event lastTimestamp: "2026-07-02T14:36:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:36:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:25Z" name: success-200-isvc-fbc8b-predictor-5f496dcc84-gpcr7.18be7fc4a4d8f710 namespace: kserve-ci-e2e-test resourceVersion: "19517" uid: aa855527-1a14-42f2-a108-75301ae05d88 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fbc8b-predictor-5f496dcc84-gpcr7 namespace: kserve-ci-e2e-test resourceVersion: "19448" uid: 3a0e9c74-f5b3-407c-a5d5-15377a551448 kind: Event lastTimestamp: "2026-07-02T14:36:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:36:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:25Z" name: success-200-isvc-fbc8b-predictor-5f496dcc84-gpcr7.18be7fc4a552beda namespace: kserve-ci-e2e-test resourceVersion: "19518" uid: 1b3da974-f59e-4143-8a5a-01fc4f1c9ee4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:36:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fbc8b-predictor-5f496dcc84-gpcr7 namespace: kserve-ci-e2e-test resourceVersion: "19448" uid: 3a0e9c74-f5b3-407c-a5d5-15377a551448 kind: Event lastTimestamp: "2026-07-02T14:37:01Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:36:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:01Z" name: success-200-isvc-fbc8b-predictor-5f496dcc84-gpcr7.18be7fc4b9f9fe96 namespace: kserve-ci-e2e-test resourceVersion: "19822" uid: 4690c534-56c1-49fb-a8ea-700803434e3e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fbc8b-predictor-5f496dcc84-gpcr7 namespace: kserve-ci-e2e-test resourceVersion: "19448" uid: 3a0e9c74-f5b3-407c-a5d5-15377a551448 kind: Event lastTimestamp: "2026-07-02T14:37:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:34Z" name: success-200-isvc-fbc8b-predictor-5f496dcc84-gpcr7.18be7fd4dc82d3c1 namespace: kserve-ci-e2e-test resourceVersion: "20322" uid: 1cc82fe8-af25-44af-9a69-a955a5633944 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fbc8b-predictor-5f496dcc84-gpcr7 namespace: kserve-ci-e2e-test resourceVersion: "19448" uid: 3a0e9c74-f5b3-407c-a5d5-15377a551448 kind: Event lastTimestamp: "2026-07-02T14:37:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:34Z" name: success-200-isvc-fbc8b-predictor-5f496dcc84-gpcr7.18be7fd4dc88eeac namespace: kserve-ci-e2e-test resourceVersion: "20324" uid: 6b0013f7-0167-492c-a13d-98d1a02fc310 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fbc8b-predictor-5f496dcc84-gpcr7 namespace: kserve-ci-e2e-test resourceVersion: "19448" uid: 3a0e9c74-f5b3-407c-a5d5-15377a551448 kind: Event lastTimestamp: "2026-07-02T14:37:36Z" message: 'Readiness probe failed: Get "https://10.133.0.28:8643/healthz": dial tcp 10.133.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:37:36Z" name: success-200-isvc-fbc8b-predictor-5f496dcc84-gpcr7.18be7fd542860d70 namespace: kserve-ci-e2e-test resourceVersion: "20424" uid: 9c41ca90-3c00-4101-899d-095f3ed991ed reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-54.ec2.internal source: component: kubelet host: ip-10-0-134-54.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-fbc8b-predictor-5f496dcc84 namespace: kserve-ci-e2e-test resourceVersion: "19428" uid: f955cb1c-c5c0-40d9-8005-afa2b423f937 kind: Event lastTimestamp: "2026-07-02T14:36:24Z" message: 'Created pod: success-200-isvc-fbc8b-predictor-5f496dcc84-gpcr7' metadata: creationTimestamp: "2026-07-02T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:36:24Z" name: success-200-isvc-fbc8b-predictor-5f496dcc84.18be7fc4823c2b26 namespace: kserve-ci-e2e-test resourceVersion: "19447" uid: 0b8be160-5495-43f4-9ce5-0470448933be reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-fbc8b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19425" uid: 51e3068a-fb67-4947-9489-1fa7e8d467be kind: Event lastTimestamp: "2026-07-02T14:36:24Z" message: Scaled up replica set success-200-isvc-fbc8b-predictor-5f496dcc84 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:36:24Z" name: success-200-isvc-fbc8b-predictor.18be7fc47e95201a namespace: kserve-ci-e2e-test resourceVersion: "19434" uid: d125e075-12fa-4278-b0ee-267e9e70f1f0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:36:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fbc8b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19427" uid: a87e4ae1-15cd-430a-9099-ceaf0b3daa80 kind: Event lastTimestamp: "2026-07-02T14:36:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:36:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:36:54Z" name: success-200-isvc-fbc8b-predictor.18be7fc7fcc349cd namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: 0dd4dafe-4440-4ff1-8570-1b837951c328 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:36:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fbc8b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19427" uid: a87e4ae1-15cd-430a-9099-ceaf0b3daa80 kind: Event lastTimestamp: "2026-07-02T14:36:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:36:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:36:54Z" name: success-200-isvc-fbc8b-predictor.18be7fc7fd2a0ac6 namespace: kserve-ci-e2e-test resourceVersion: "19770" uid: 5babeddd-e8bf-4962-8981-8e64fbae5c94 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:37:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fbc8b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19653" uid: a87e4ae1-15cd-430a-9099-ceaf0b3daa80 kind: Event lastTimestamp: "2026-07-02T14:37:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:37:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:37:24Z" name: success-200-isvc-fbc8b-predictor.18be7fcefa7fc2f4 namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: 796af1fa-b465-48a3-a37b-86e7844cd770 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:37:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fbc8b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19653" uid: a87e4ae1-15cd-430a-9099-ceaf0b3daa80 kind: Event lastTimestamp: "2026-07-02T14:37:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:37:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:37:24Z" name: success-200-isvc-fbc8b-predictor.18be7fcefaec2f3f namespace: kserve-ci-e2e-test resourceVersion: "20161" uid: 75428e22-1e98-44cd-aef3-80375b74e232 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:36:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fbc8b namespace: kserve-ci-e2e-test resourceVersion: "19412" uid: 1afda277-4353-4086-a922-5c31522468e3 kind: Event lastTimestamp: "2026-07-02T14:37:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:37:11Z" name: success-200-isvc-fbc8b.18be7fc47ebe6cd1 namespace: kserve-ci-e2e-test resourceVersion: "19905" uid: a5d63726-dfb6-46b8-bf52-558209a9c5e8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fbc8b namespace: kserve-ci-e2e-test resourceVersion: "19412" uid: 1afda277-4353-4086-a922-5c31522468e3 kind: Event lastTimestamp: "2026-07-02T14:36:24Z" message: 'Failed to update status for InferenceService "success-200-isvc-fbc8b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-fbc8b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:36:24Z" name: success-200-isvc-fbc8b.18be7fc47f66653f namespace: kserve-ci-e2e-test resourceVersion: "19439" uid: 6b5e499c-51ac-4e78-bcb1-adcd0a0ea47c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fbc8b namespace: kserve-ci-e2e-test resourceVersion: "19412" uid: 1afda277-4353-4086-a922-5c31522468e3 kind: Event lastTimestamp: "2026-07-02T14:36:24Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-fbc8b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:36:24Z" name: success-200-isvc-fbc8b.18be7fc47f66bece namespace: kserve-ci-e2e-test resourceVersion: "19442" uid: fc9eeac4-f27d-4ec8-a694-b080ba216488 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:37:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fbc8b namespace: kserve-ci-e2e-test resourceVersion: "19903" uid: 1afda277-4353-4086-a922-5c31522468e3 kind: Event lastTimestamp: "2026-07-02T14:37:11Z" message: InferenceService [success-200-isvc-fbc8b] is Ready metadata: creationTimestamp: "2026-07-02T14:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:37:11Z" name: success-200-isvc-fbc8b.18be7fcf75e40233 namespace: kserve-ci-e2e-test resourceVersion: "19904" uid: dc93ef8c-3856-454e-9d2a-7be36c6ce98f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:28:09.496775Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1c2b7-5f799fb869-vb6lr namespace: kserve-ci-e2e-test resourceVersion: "15868" uid: 562208bb-2466-484f-a77b-4d413085093a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-1c2b7-5f799fb869-vb6lr to ip-10-0-143-249.ec2.internal metadata: creationTimestamp: "2026-07-02T14:28:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:28:09Z" name: switch-graph-1c2b7-5f799fb869-vb6lr.18be7f514417cfea namespace: kserve-ci-e2e-test resourceVersion: "15873" uid: f099c375-78c6-4a50-bdb0-44537aab3181 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:09Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1c2b7-5f799fb869-vb6lr namespace: kserve-ci-e2e-test resourceVersion: "15870" uid: 562208bb-2466-484f-a77b-4d413085093a kind: Event lastTimestamp: "2026-07-02T14:28:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-1c2b7-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:09Z" name: switch-graph-1c2b7-5f799fb869-vb6lr.18be7f514ec37a28 namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: d7362521-fed0-417c-a3f2-71e830ed9a8d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:10Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1c2b7-5f799fb869-vb6lr namespace: kserve-ci-e2e-test resourceVersion: "15876" uid: 562208bb-2466-484f-a77b-4d413085093a kind: Event lastTimestamp: "2026-07-02T14:28:10Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:28:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:28:10Z" name: switch-graph-1c2b7-5f799fb869-vb6lr.18be7f5181251b2a namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 8f9e18fd-2235-473c-8342-3e35400d28ab reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1c2b7} kind: Pod name: switch-graph-1c2b7-5f799fb869-vb6lr namespace: kserve-ci-e2e-test resourceVersion: "15870" uid: 562208bb-2466-484f-a77b-4d413085093a kind: Event lastTimestamp: "2026-07-02T14:28:10Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:e0bdb71677d5eb7ee4784378e591e8126d0dcafbbe991e5dcf178c67e85889cd" already present on machine metadata: creationTimestamp: "2026-07-02T14:28:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:10Z" name: switch-graph-1c2b7-5f799fb869-vb6lr.18be7f5182ca0841 namespace: kserve-ci-e2e-test resourceVersion: "15901" uid: e6561d03-4a3d-480e-85bd-a348b0c77a67 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1c2b7} kind: Pod name: switch-graph-1c2b7-5f799fb869-vb6lr namespace: kserve-ci-e2e-test resourceVersion: "15870" uid: 562208bb-2466-484f-a77b-4d413085093a kind: Event lastTimestamp: "2026-07-02T14:28:10Z" message: 'Created container: switch-graph-1c2b7' metadata: creationTimestamp: "2026-07-02T14:28:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:10Z" name: switch-graph-1c2b7-5f799fb869-vb6lr.18be7f5185bf84cd namespace: kserve-ci-e2e-test resourceVersion: "15902" uid: 0fb8e910-12f7-4c57-b92e-239a6d153618 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1c2b7} kind: Pod name: switch-graph-1c2b7-5f799fb869-vb6lr namespace: kserve-ci-e2e-test resourceVersion: "15870" uid: 562208bb-2466-484f-a77b-4d413085093a kind: Event lastTimestamp: "2026-07-02T14:28:10Z" message: Started container switch-graph-1c2b7 metadata: creationTimestamp: "2026-07-02T14:28:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:28:10Z" name: switch-graph-1c2b7-5f799fb869-vb6lr.18be7f51863ae34c namespace: kserve-ci-e2e-test resourceVersion: "15903" uid: 86a78be7-41dd-429a-88a5-4ac8acb8a80d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:36:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1c2b7} kind: Pod name: switch-graph-1c2b7-5f799fb869-vb6lr namespace: kserve-ci-e2e-test resourceVersion: "15870" uid: 562208bb-2466-484f-a77b-4d413085093a kind: Event lastTimestamp: "2026-07-02T14:36:24Z" message: Stopping container switch-graph-1c2b7 metadata: creationTimestamp: "2026-07-02T14:36:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:24Z" name: switch-graph-1c2b7-5f799fb869-vb6lr.18be7fc475f2221d namespace: kserve-ci-e2e-test resourceVersion: "19406" uid: 90e79adc-f3d8-43f6-8675-aa1614e31378 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:36:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1c2b7} kind: Pod name: switch-graph-1c2b7-5f799fb869-vb6lr namespace: kserve-ci-e2e-test resourceVersion: "15870" uid: 562208bb-2466-484f-a77b-4d413085093a kind: Event lastTimestamp: "2026-07-02T14:36:52Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:36:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:36:52Z" name: switch-graph-1c2b7-5f799fb869-vb6lr.18be7fc52039bf6e namespace: kserve-ci-e2e-test resourceVersion: "19753" uid: 8a8e4274-898d-4994-a393-16eda31b5ea9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-1c2b7-5f799fb869 namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 58186184-bca9-4fb3-953e-a9121e222acd kind: Event lastTimestamp: "2026-07-02T14:28:09Z" message: 'Created pod: switch-graph-1c2b7-5f799fb869-vb6lr' metadata: creationTimestamp: "2026-07-02T14:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:09Z" name: switch-graph-1c2b7-5f799fb869.18be7f51439dadaa namespace: kserve-ci-e2e-test resourceVersion: "15869" uid: af614a5b-e27d-43f2-ba27-f660d59cde88 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-1c2b7 namespace: kserve-ci-e2e-test resourceVersion: "15859" uid: 07267741-d846-42cd-84fc-1436ec94af06 kind: Event lastTimestamp: "2026-07-02T14:28:09Z" message: Scaled up replica set switch-graph-1c2b7-5f799fb869 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:09Z" name: switch-graph-1c2b7.18be7f5142b1475f namespace: kserve-ci-e2e-test resourceVersion: "15864" uid: 3cf7fac9-6e88-4937-b44a-faa3112a8042 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:09Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-1c2b7 namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: 7882a1b3-29eb-44c8-af0e-90f1471579ce kind: Event lastTimestamp: "2026-07-02T14:28:09Z" message: InferenceGraph [switch-graph-1c2b7] is Ready metadata: creationTimestamp: "2026-07-02T14:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:28:09Z" name: switch-graph-1c2b7.18be7f5149162462 namespace: kserve-ci-e2e-test resourceVersion: "15884" uid: 586c74c5-07f4-42e5-998c-c943359ab226 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-1c2b7 namespace: kserve-ci-e2e-test resourceVersion: "15861" uid: cf4f6bfc-2104-4ed3-a10b-b1a53bb96863 kind: Event lastTimestamp: "2026-07-02T14:28:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:28:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:24Z" name: switch-graph-1c2b7.18be7f54c125b75f namespace: kserve-ci-e2e-test resourceVersion: "16032" uid: 156675b9-c1b2-4b9c-9780-643ae776da0a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-1c2b7 namespace: kserve-ci-e2e-test resourceVersion: "15861" uid: cf4f6bfc-2104-4ed3-a10b-b1a53bb96863 kind: Event lastTimestamp: "2026-07-02T14:28:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:28:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:24Z" name: switch-graph-1c2b7.18be7f54c18937a2 namespace: kserve-ci-e2e-test resourceVersion: "16033" uid: 33c11de6-2fa2-4115-b5ae-425383ab5414 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-1c2b7 namespace: kserve-ci-e2e-test resourceVersion: "16030" uid: cf4f6bfc-2104-4ed3-a10b-b1a53bb96863 kind: Event lastTimestamp: "2026-07-02T14:28:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:28:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:39Z" name: switch-graph-1c2b7.18be7f5840240128 namespace: kserve-ci-e2e-test resourceVersion: "16151" uid: b53eac61-8a0c-4875-a0c9-8cd3ee6bd7c4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:28:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-1c2b7 namespace: kserve-ci-e2e-test resourceVersion: "16030" uid: cf4f6bfc-2104-4ed3-a10b-b1a53bb96863 kind: Event lastTimestamp: "2026-07-02T14:28:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:28:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:28:39Z" name: switch-graph-1c2b7.18be7f5840822ac1 namespace: kserve-ci-e2e-test resourceVersion: "16152" uid: ce2dba28-197c-4b9d-bd07-a264c443625b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:49:09.487061Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-cb06c-565f58d8db-t6t92 namespace: kserve-ci-e2e-test resourceVersion: "26601" uid: 76f73dd7-f380-41ed-ab11-44c557a263a6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-cb06c-565f58d8db-t6t92 to ip-10-0-143-249.ec2.internal metadata: creationTimestamp: "2026-07-02T14:49:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:49:09Z" name: switch-graph-cb06c-565f58d8db-t6t92.18be8076a15db85c namespace: kserve-ci-e2e-test resourceVersion: "26607" uid: 986b781d-9486-48d2-99a7-86313531f5ea reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:49:09Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-cb06c-565f58d8db-t6t92 namespace: kserve-ci-e2e-test resourceVersion: "26611" uid: 76f73dd7-f380-41ed-ab11-44c557a263a6 kind: Event lastTimestamp: "2026-07-02T14:49:09Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:49:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:49:09Z" name: switch-graph-cb06c-565f58d8db-t6t92.18be8076baeed738 namespace: kserve-ci-e2e-test resourceVersion: "26622" uid: caf4cdfc-3ccc-4e60-9096-a939c5486735 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:49:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-cb06c} kind: Pod name: switch-graph-cb06c-565f58d8db-t6t92 namespace: kserve-ci-e2e-test resourceVersion: "26605" uid: 76f73dd7-f380-41ed-ab11-44c557a263a6 kind: Event lastTimestamp: "2026-07-02T14:49:09Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:e0bdb71677d5eb7ee4784378e591e8126d0dcafbbe991e5dcf178c67e85889cd" already present on machine metadata: creationTimestamp: "2026-07-02T14:49:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:49:09Z" name: switch-graph-cb06c-565f58d8db-t6t92.18be8076bc4d0047 namespace: kserve-ci-e2e-test resourceVersion: "26625" uid: 2e1a70ff-af26-4b5e-932e-b717d30cf7c3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:49:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-cb06c} kind: Pod name: switch-graph-cb06c-565f58d8db-t6t92 namespace: kserve-ci-e2e-test resourceVersion: "26605" uid: 76f73dd7-f380-41ed-ab11-44c557a263a6 kind: Event lastTimestamp: "2026-07-02T14:49:09Z" message: 'Created container: switch-graph-cb06c' metadata: creationTimestamp: "2026-07-02T14:49:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:49:09Z" name: switch-graph-cb06c-565f58d8db-t6t92.18be8076bf444cbc namespace: kserve-ci-e2e-test resourceVersion: "26637" uid: 5bcdcc23-a62d-4c66-bee1-ad58161a61fb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:49:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-cb06c} kind: Pod name: switch-graph-cb06c-565f58d8db-t6t92 namespace: kserve-ci-e2e-test resourceVersion: "26605" uid: 76f73dd7-f380-41ed-ab11-44c557a263a6 kind: Event lastTimestamp: "2026-07-02T14:49:09Z" message: Started container switch-graph-cb06c metadata: creationTimestamp: "2026-07-02T14:49:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:49:09Z" name: switch-graph-cb06c-565f58d8db-t6t92.18be8076bfb42948 namespace: kserve-ci-e2e-test resourceVersion: "26641" uid: 7a28dd28-a144-4c59-9533-3cb5a65f49ad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T15:05:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-cb06c} kind: Pod name: switch-graph-cb06c-565f58d8db-t6t92 namespace: kserve-ci-e2e-test resourceVersion: "26605" uid: 76f73dd7-f380-41ed-ab11-44c557a263a6 kind: Event lastTimestamp: "2026-07-02T15:05:28Z" message: Stopping container switch-graph-cb06c metadata: creationTimestamp: "2026-07-02T15:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T15:05:28Z" name: switch-graph-cb06c-565f58d8db-t6t92.18be815aa70551cd namespace: kserve-ci-e2e-test resourceVersion: "33493" uid: 7bf70b97-213e-417f-ba35-6f470da26268 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T15:05:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-cb06c} kind: Pod name: switch-graph-cb06c-565f58d8db-t6t92 namespace: kserve-ci-e2e-test resourceVersion: "26605" uid: 76f73dd7-f380-41ed-ab11-44c557a263a6 kind: Event lastTimestamp: "2026-07-02T15:05:56Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T15:05:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T15:05:56Z" name: switch-graph-cb06c-565f58d8db-t6t92.18be815b39c7c02d namespace: kserve-ci-e2e-test resourceVersion: "33765" uid: 314b47b1-8d09-4c3d-a614-6848125bef97 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:49:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-cb06c-565f58d8db namespace: kserve-ci-e2e-test resourceVersion: "26597" uid: fc104f1d-f5e4-409a-b18e-a43230c4050d kind: Event lastTimestamp: "2026-07-02T14:49:09Z" message: 'Created pod: switch-graph-cb06c-565f58d8db-t6t92' metadata: creationTimestamp: "2026-07-02T14:49:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:49:09Z" name: switch-graph-cb06c-565f58d8db.18be8076a0f1806b namespace: kserve-ci-e2e-test resourceVersion: "26604" uid: 1589cc55-5c3f-4fa5-8eee-7f163eea6b37 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:49:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-cb06c namespace: kserve-ci-e2e-test resourceVersion: "26595" uid: 97326a4b-e4e9-4bd3-80df-a3291ccb453e kind: Event lastTimestamp: "2026-07-02T14:49:09Z" message: Scaled up replica set switch-graph-cb06c-565f58d8db from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:49:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:49:09Z" name: switch-graph-cb06c.18be8076a0183038 namespace: kserve-ci-e2e-test resourceVersion: "26598" uid: 9402b49f-d64d-4f5a-b569-0e667c727e43 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:49:09Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-cb06c namespace: kserve-ci-e2e-test resourceVersion: "26613" uid: a040700b-2e56-4faf-901a-81afb49724de kind: Event lastTimestamp: "2026-07-02T14:49:09Z" message: InferenceGraph [switch-graph-cb06c] is Ready metadata: creationTimestamp: "2026-07-02T14:49:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:49:09Z" name: switch-graph-cb06c.18be8076a3c1ae05 namespace: kserve-ci-e2e-test resourceVersion: "26615" uid: 056e5520-ecd3-4319-856b-398c9f51d450 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:49:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-cb06c namespace: kserve-ci-e2e-test resourceVersion: "26596" uid: b69330cd-92b3-451e-beab-8221330af0e2 kind: Event lastTimestamp: "2026-07-02T14:49:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:49:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:49:24Z" name: switch-graph-cb06c.18be807a1f0a9e01 namespace: kserve-ci-e2e-test resourceVersion: "26783" uid: 1c46a2ca-4352-4ba1-8b07-7781fd886f90 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:49:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-cb06c namespace: kserve-ci-e2e-test resourceVersion: "26596" uid: b69330cd-92b3-451e-beab-8221330af0e2 kind: Event lastTimestamp: "2026-07-02T14:49:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:49:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:49:24Z" name: switch-graph-cb06c.18be807a1f5843ee namespace: kserve-ci-e2e-test resourceVersion: "26784" uid: 2b7cabb2-58a8-438e-b33b-6dc5a0a83e47 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:49:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-cb06c namespace: kserve-ci-e2e-test resourceVersion: "26782" uid: b69330cd-92b3-451e-beab-8221330af0e2 kind: Event lastTimestamp: "2026-07-02T14:49:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:49:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:49:39Z" name: switch-graph-cb06c.18be807d9e094ac9 namespace: kserve-ci-e2e-test resourceVersion: "26896" uid: d92ac7ca-167b-4d7d-a276-8a1e0eec79b0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:49:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-cb06c namespace: kserve-ci-e2e-test resourceVersion: "26782" uid: b69330cd-92b3-451e-beab-8221330af0e2 kind: Event lastTimestamp: "2026-07-02T14:49:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:49:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:49:39Z" name: switch-graph-cb06c.18be807d9e7c7e12 namespace: kserve-ci-e2e-test resourceVersion: "26899" uid: acfaf992-bdf9-45c9-a4cf-1448a12549aa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:26:55.124258Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-df2c1-84fd8ddb75-snq4d namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 66798a0f-7d3f-4bfa-a9c9-c7fac5fa54e1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-df2c1-84fd8ddb75-snq4d to ip-10-0-143-249.ec2.internal metadata: creationTimestamp: "2026-07-02T14:26:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:26:55Z" name: switch-graph-df2c1-84fd8ddb75-snq4d.18be7f3ff325607f namespace: kserve-ci-e2e-test resourceVersion: "14846" uid: 333e07fc-cf2e-4f01-9c1e-e03850da942c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-bfbc6869c-jjtck source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:26:55Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-df2c1-84fd8ddb75-snq4d namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 66798a0f-7d3f-4bfa-a9c9-c7fac5fa54e1 kind: Event lastTimestamp: "2026-07-02T14:26:55Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-df2c1-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:55Z" name: switch-graph-df2c1-84fd8ddb75-snq4d.18be7f3ffc6ec779 namespace: kserve-ci-e2e-test resourceVersion: "14861" uid: 350c9575-7ca0-4d94-acd2-8fd1792a03d1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:57Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-df2c1-84fd8ddb75-snq4d namespace: kserve-ci-e2e-test resourceVersion: "14850" uid: 66798a0f-7d3f-4bfa-a9c9-c7fac5fa54e1 kind: Event lastTimestamp: "2026-07-02T14:26:57Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:26:57Z" name: switch-graph-df2c1-84fd8ddb75-snq4d.18be7f4065a4a3e0 namespace: kserve-ci-e2e-test resourceVersion: "14871" uid: fd2925dd-b604-436a-880a-cac9964b169d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-df2c1} kind: Pod name: switch-graph-df2c1-84fd8ddb75-snq4d namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 66798a0f-7d3f-4bfa-a9c9-c7fac5fa54e1 kind: Event lastTimestamp: "2026-07-02T14:26:57Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:e0bdb71677d5eb7ee4784378e591e8126d0dcafbbe991e5dcf178c67e85889cd" metadata: creationTimestamp: "2026-07-02T14:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:57Z" name: switch-graph-df2c1-84fd8ddb75-snq4d.18be7f4067021026 namespace: kserve-ci-e2e-test resourceVersion: "14873" uid: ceb97173-d43e-4c4c-9827-2630e4d21744 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-df2c1} kind: Pod name: switch-graph-df2c1-84fd8ddb75-snq4d namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 66798a0f-7d3f-4bfa-a9c9-c7fac5fa54e1 kind: Event lastTimestamp: "2026-07-02T14:26:59Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:e0bdb71677d5eb7ee4784378e591e8126d0dcafbbe991e5dcf178c67e85889cd" in 2.84s (2.84s including waiting). Image size: 218068126 bytes.' metadata: creationTimestamp: "2026-07-02T14:26:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:59Z" name: switch-graph-df2c1-84fd8ddb75-snq4d.18be7f41104b6225 namespace: kserve-ci-e2e-test resourceVersion: "14899" uid: 7a6274e4-b593-45c2-a95b-2bfd00bc68c1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-df2c1} kind: Pod name: switch-graph-df2c1-84fd8ddb75-snq4d namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 66798a0f-7d3f-4bfa-a9c9-c7fac5fa54e1 kind: Event lastTimestamp: "2026-07-02T14:26:59Z" message: 'Created container: switch-graph-df2c1' metadata: creationTimestamp: "2026-07-02T14:26:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:59Z" name: switch-graph-df2c1-84fd8ddb75-snq4d.18be7f41139aeece namespace: kserve-ci-e2e-test resourceVersion: "14900" uid: c9858166-8aa2-48fd-aa48-a9f2cad8c7b1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-df2c1} kind: Pod name: switch-graph-df2c1-84fd8ddb75-snq4d namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 66798a0f-7d3f-4bfa-a9c9-c7fac5fa54e1 kind: Event lastTimestamp: "2026-07-02T14:26:59Z" message: Started container switch-graph-df2c1 metadata: creationTimestamp: "2026-07-02T14:26:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:26:59Z" name: switch-graph-df2c1-84fd8ddb75-snq4d.18be7f41140d1e44 namespace: kserve-ci-e2e-test resourceVersion: "14901" uid: 5869135d-d042-4276-a1f9-cccdb65b537c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:27:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-df2c1} kind: Pod name: switch-graph-df2c1-84fd8ddb75-snq4d namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 66798a0f-7d3f-4bfa-a9c9-c7fac5fa54e1 kind: Event lastTimestamp: "2026-07-02T14:27:09Z" message: Stopping container switch-graph-df2c1 metadata: creationTimestamp: "2026-07-02T14:27:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:09Z" name: switch-graph-df2c1-84fd8ddb75-snq4d.18be7f433d6c966e namespace: kserve-ci-e2e-test resourceVersion: "15004" uid: 8b116b75-41f5-4479-81cd-62c2d6415466 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:27:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-df2c1} kind: Pod name: switch-graph-df2c1-84fd8ddb75-snq4d namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 66798a0f-7d3f-4bfa-a9c9-c7fac5fa54e1 kind: Event lastTimestamp: "2026-07-02T14:27:36Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:27:36Z" name: switch-graph-df2c1-84fd8ddb75-snq4d.18be7f43d602c0d3 namespace: kserve-ci-e2e-test resourceVersion: "15382" uid: 056401b4-7990-4e0e-b3d4-dec8a6476745 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-249.ec2.internal source: component: kubelet host: ip-10-0-143-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-df2c1-84fd8ddb75 namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: 58db9973-89a0-49f9-9e7c-57a67459504c kind: Event lastTimestamp: "2026-07-02T14:26:55Z" message: 'Created pod: switch-graph-df2c1-84fd8ddb75-snq4d' metadata: creationTimestamp: "2026-07-02T14:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:26:55Z" name: switch-graph-df2c1-84fd8ddb75.18be7f3ff2998935 namespace: kserve-ci-e2e-test resourceVersion: "14843" uid: 1adf9d9e-a36b-4003-9ecd-298f466ccb0a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-df2c1 namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: 733beea8-f3da-45eb-ab81-1a1a24a938e1 kind: Event lastTimestamp: "2026-07-02T14:26:55Z" message: Scaled up replica set switch-graph-df2c1-84fd8ddb75 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:26:55Z" name: switch-graph-df2c1.18be7f3ff186dd84 namespace: kserve-ci-e2e-test resourceVersion: "14839" uid: daced754-e8fa-4b71-976a-a5db80256f36 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:26:55Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-df2c1 namespace: kserve-ci-e2e-test resourceVersion: "14852" uid: d7763af0-9b96-491b-88fe-c712b8ca1b28 kind: Event lastTimestamp: "2026-07-02T14:26:55Z" message: InferenceGraph [switch-graph-df2c1] is Ready metadata: creationTimestamp: "2026-07-02T14:26:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:26:55Z" name: switch-graph-df2c1.18be7f3ff5643533 namespace: kserve-ci-e2e-test resourceVersion: "14854" uid: aca0f50d-93fd-4cdc-9f27-6fdced996730 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal kind: EventList metadata: resourceVersion: "33768"