--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:56:34.041438Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-3f928-6dc74cd58b-jw88j namespace: kserve-ci-e2e-test resourceVersion: "21570" uid: 93c59413-c05b-43b0-b7d9-69de5d9d8344 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-3f928-6dc74cd58b-jw88j to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T15:56:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:56:34Z" name: ensemble-graph-3f928-6dc74cd58b-jw88j.18b1512feb52db9d namespace: kserve-ci-e2e-test resourceVersion: "21575" uid: c8ef8938-6cbf-443d-b17d-00bb22df1cd5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:56:34Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-3f928-6dc74cd58b-jw88j namespace: kserve-ci-e2e-test resourceVersion: "21579" uid: 93c59413-c05b-43b0-b7d9-69de5d9d8344 kind: Event lastTimestamp: "2026-05-20T15:56:34Z" message: Add eth0 [10.132.0.56/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:56:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:56:34Z" name: ensemble-graph-3f928-6dc74cd58b-jw88j.18b1513005699fbf namespace: kserve-ci-e2e-test resourceVersion: "21591" uid: fb8076b0-1279-458c-bd87-725266bdd4e0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:56:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3f928} kind: Pod name: ensemble-graph-3f928-6dc74cd58b-jw88j namespace: kserve-ci-e2e-test resourceVersion: "21572" uid: 93c59413-c05b-43b0-b7d9-69de5d9d8344 kind: Event lastTimestamp: "2026-05-20T15:56:34Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:88955c235550cb8e8a9cbe850692d2683b6dea68a08743540305f46e732a59a3" already present on machine metadata: creationTimestamp: "2026-05-20T15:56:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:56:34Z" name: ensemble-graph-3f928-6dc74cd58b-jw88j.18b1513006c2ba71 namespace: kserve-ci-e2e-test resourceVersion: "21594" uid: fa0ab9d5-e776-488d-ba9b-d3885d869e32 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:56:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3f928} kind: Pod name: ensemble-graph-3f928-6dc74cd58b-jw88j namespace: kserve-ci-e2e-test resourceVersion: "21572" uid: 93c59413-c05b-43b0-b7d9-69de5d9d8344 kind: Event lastTimestamp: "2026-05-20T15:56:34Z" message: 'Created container: ensemble-graph-3f928' metadata: creationTimestamp: "2026-05-20T15:56:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:56:34Z" name: ensemble-graph-3f928-6dc74cd58b-jw88j.18b151300a5ab167 namespace: kserve-ci-e2e-test resourceVersion: "21595" uid: af534efb-fe00-4d7a-9deb-5cdee4174c00 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:56:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3f928} kind: Pod name: ensemble-graph-3f928-6dc74cd58b-jw88j namespace: kserve-ci-e2e-test resourceVersion: "21572" uid: 93c59413-c05b-43b0-b7d9-69de5d9d8344 kind: Event lastTimestamp: "2026-05-20T15:56:34Z" message: Started container ensemble-graph-3f928 metadata: creationTimestamp: "2026-05-20T15:56:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:56:34Z" name: ensemble-graph-3f928-6dc74cd58b-jw88j.18b151300ad9d5c0 namespace: kserve-ci-e2e-test resourceVersion: "21596" uid: 1d44784e-5493-4005-a4d3-1d253a04bc4d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3f928} kind: Pod name: ensemble-graph-3f928-6dc74cd58b-jw88j namespace: kserve-ci-e2e-test resourceVersion: "21572" uid: 93c59413-c05b-43b0-b7d9-69de5d9d8344 kind: Event lastTimestamp: "2026-05-20T16:04:48Z" message: Stopping container ensemble-graph-3f928 metadata: creationTimestamp: "2026-05-20T16:04:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:04:48Z" name: ensemble-graph-3f928-6dc74cd58b-jw88j.18b151a31aef25e9 namespace: kserve-ci-e2e-test resourceVersion: "25335" uid: 97006f7e-31db-4f8d-bcad-691139081c6a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T16:04:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3f928} kind: Pod name: ensemble-graph-3f928-6dc74cd58b-jw88j namespace: kserve-ci-e2e-test resourceVersion: "21572" uid: 93c59413-c05b-43b0-b7d9-69de5d9d8344 kind: Event lastTimestamp: "2026-05-20T16:05:15Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T16:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:15Z" name: ensemble-graph-3f928-6dc74cd58b-jw88j.18b151a39f24bfe0 namespace: kserve-ci-e2e-test resourceVersion: "25829" uid: eba32981-52a0-4fef-80d0-ead2adbc57eb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:56:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-3f928-6dc74cd58b namespace: kserve-ci-e2e-test resourceVersion: "21567" uid: 05b9bd6e-e388-4bfc-8e08-a264772adac0 kind: Event lastTimestamp: "2026-05-20T15:56:34Z" message: 'Created pod: ensemble-graph-3f928-6dc74cd58b-jw88j' metadata: creationTimestamp: "2026-05-20T15:56:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:56:34Z" name: ensemble-graph-3f928-6dc74cd58b.18b1512feaecc2d9 namespace: kserve-ci-e2e-test resourceVersion: "21571" uid: 64278826-53de-47f9-8cd0-13d6632d5d5e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:56:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-3f928 namespace: kserve-ci-e2e-test resourceVersion: "21565" uid: fddfb1ee-16ea-4a48-aa68-57b5767c3407 kind: Event lastTimestamp: "2026-05-20T15:56:34Z" message: Scaled up replica set ensemble-graph-3f928-6dc74cd58b from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:56:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:56:34Z" name: ensemble-graph-3f928.18b1512fea2064bd namespace: kserve-ci-e2e-test resourceVersion: "21568" uid: 694dd308-5ee0-4609-8fb6-9c42086dc3bf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:56:34Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-3f928 namespace: kserve-ci-e2e-test resourceVersion: "21582" uid: b73e89b4-6a26-48d7-9a7d-97729a0c4e7a kind: Event lastTimestamp: "2026-05-20T15:56:34Z" message: InferenceGraph [ensemble-graph-3f928] is Ready metadata: creationTimestamp: "2026-05-20T15:56:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:56:34Z" name: ensemble-graph-3f928.18b1512fee9de85a namespace: kserve-ci-e2e-test resourceVersion: "21585" uid: 0891336d-e42f-40a8-a09a-5b2a81010c6a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:56:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-3f928 namespace: kserve-ci-e2e-test resourceVersion: "21566" uid: f769867e-ad7c-4a05-a250-3d192abc997b kind: Event lastTimestamp: "2026-05-20T15:56:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:56:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:56:49Z" name: ensemble-graph-3f928.18b1513368afbd73 namespace: kserve-ci-e2e-test resourceVersion: "21726" uid: dcbda74d-bfe0-45d8-b998-c12c4a0d5a23 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:56:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-3f928 namespace: kserve-ci-e2e-test resourceVersion: "21566" uid: f769867e-ad7c-4a05-a250-3d192abc997b kind: Event lastTimestamp: "2026-05-20T15:56:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:56:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:56:49Z" name: ensemble-graph-3f928.18b15133691323d9 namespace: kserve-ci-e2e-test resourceVersion: "21728" uid: ce26a44a-7f93-4811-bdb6-c490b7040ccd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:57:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-3f928 namespace: kserve-ci-e2e-test resourceVersion: "21727" uid: f769867e-ad7c-4a05-a250-3d192abc997b kind: Event lastTimestamp: "2026-05-20T15:57:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:57:04Z" name: ensemble-graph-3f928.18b15136e7e6be95 namespace: kserve-ci-e2e-test resourceVersion: "21885" uid: 5b03bfdb-ee0d-404c-bdfa-052956c924f0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:57:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-3f928 namespace: kserve-ci-e2e-test resourceVersion: "21727" uid: f769867e-ad7c-4a05-a250-3d192abc997b kind: Event lastTimestamp: "2026-05-20T15:57:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:57:04Z" name: ensemble-graph-3f928.18b15136e8363dc7 namespace: kserve-ci-e2e-test resourceVersion: "21887" uid: 6c61e901-bf57-4712-be50-3dc7eb8cdaca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:55:13.774484Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-687f8-5b9ff78587-xmm2c namespace: kserve-ci-e2e-test resourceVersion: "20520" uid: 13eb4007-0f04-4f0d-8768-7a3de2ab17e4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-687f8-5b9ff78587-xmm2c to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T15:55:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:55:13Z" name: ensemble-graph-687f8-5b9ff78587-xmm2c.18b1511d3b0a5b13 namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: d164b6c9-d853-4749-98ca-c1ae243ccfb1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:13Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-687f8-5b9ff78587-xmm2c namespace: kserve-ci-e2e-test resourceVersion: "20522" uid: 13eb4007-0f04-4f0d-8768-7a3de2ab17e4 kind: Event lastTimestamp: "2026-05-20T15:55:13Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-687f8-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T15:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:13Z" name: ensemble-graph-687f8-5b9ff78587-xmm2c.18b1511d48345b01 namespace: kserve-ci-e2e-test resourceVersion: "20535" uid: 412b10b3-c569-4a81-b2ca-a3d5e747b780 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:14Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-687f8-5b9ff78587-xmm2c namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: 13eb4007-0f04-4f0d-8768-7a3de2ab17e4 kind: Event lastTimestamp: "2026-05-20T15:55:14Z" message: Add eth0 [10.132.0.50/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:55:14Z" name: ensemble-graph-687f8-5b9ff78587-xmm2c.18b1511d79759547 namespace: kserve-ci-e2e-test resourceVersion: "20548" uid: f3bad9bb-4a33-4f0f-bbaa-0ec99ee20813 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-687f8} kind: Pod name: ensemble-graph-687f8-5b9ff78587-xmm2c namespace: kserve-ci-e2e-test resourceVersion: "20522" uid: 13eb4007-0f04-4f0d-8768-7a3de2ab17e4 kind: Event lastTimestamp: "2026-05-20T15:55:14Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:88955c235550cb8e8a9cbe850692d2683b6dea68a08743540305f46e732a59a3" already present on machine metadata: creationTimestamp: "2026-05-20T15:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:14Z" name: ensemble-graph-687f8-5b9ff78587-xmm2c.18b1511d7ba75b9e namespace: kserve-ci-e2e-test resourceVersion: "20550" uid: 22e1fba2-9b36-4cd6-96cd-328f8900bd58 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-687f8} kind: Pod name: ensemble-graph-687f8-5b9ff78587-xmm2c namespace: kserve-ci-e2e-test resourceVersion: "20522" uid: 13eb4007-0f04-4f0d-8768-7a3de2ab17e4 kind: Event lastTimestamp: "2026-05-20T15:55:14Z" message: 'Created container: ensemble-graph-687f8' metadata: creationTimestamp: "2026-05-20T15:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:14Z" name: ensemble-graph-687f8-5b9ff78587-xmm2c.18b1511d7fb94005 namespace: kserve-ci-e2e-test resourceVersion: "20553" uid: 69ca9ec6-2100-402f-b001-5796afd23642 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-687f8} kind: Pod name: ensemble-graph-687f8-5b9ff78587-xmm2c namespace: kserve-ci-e2e-test resourceVersion: "20522" uid: 13eb4007-0f04-4f0d-8768-7a3de2ab17e4 kind: Event lastTimestamp: "2026-05-20T15:55:14Z" message: Started container ensemble-graph-687f8 metadata: creationTimestamp: "2026-05-20T15:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:14Z" name: ensemble-graph-687f8-5b9ff78587-xmm2c.18b1511d80316ff3 namespace: kserve-ci-e2e-test resourceVersion: "20554" uid: cd0e9db5-518f-4279-87ef-9c4a4207a5e0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-687f8} kind: Pod name: ensemble-graph-687f8-5b9ff78587-xmm2c namespace: kserve-ci-e2e-test resourceVersion: "20522" uid: 13eb4007-0f04-4f0d-8768-7a3de2ab17e4 kind: Event lastTimestamp: "2026-05-20T15:55:23Z" message: Stopping container ensemble-graph-687f8 metadata: creationTimestamp: "2026-05-20T15:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:23Z" name: ensemble-graph-687f8-5b9ff78587-xmm2c.18b1511f921529c3 namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: f988b445-1315-405b-80bf-94681152f6ac reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T15:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-687f8} kind: Pod name: ensemble-graph-687f8-5b9ff78587-xmm2c namespace: kserve-ci-e2e-test resourceVersion: "20522" uid: 13eb4007-0f04-4f0d-8768-7a3de2ab17e4 kind: Event lastTimestamp: "2026-05-20T15:55:51Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T15:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:51Z" name: ensemble-graph-687f8-5b9ff78587-xmm2c.18b151203b267ecd namespace: kserve-ci-e2e-test resourceVersion: "21200" uid: ef2889d4-8752-4631-89cb-3d0a3ba15e6a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-687f8-5b9ff78587 namespace: kserve-ci-e2e-test resourceVersion: "20517" uid: 16f8860c-8790-410a-9c21-c9fb103ffa4c kind: Event lastTimestamp: "2026-05-20T15:55:13Z" message: 'Created pod: ensemble-graph-687f8-5b9ff78587-xmm2c' metadata: creationTimestamp: "2026-05-20T15:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:55:13Z" name: ensemble-graph-687f8-5b9ff78587.18b1511d3a8ea445 namespace: kserve-ci-e2e-test resourceVersion: "20521" uid: 60eedc75-9265-4bc1-917c-f388e65c1f94 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-687f8 namespace: kserve-ci-e2e-test resourceVersion: "20515" uid: 5d6492ca-f617-4e75-bfbb-d1efcbe9b919 kind: Event lastTimestamp: "2026-05-20T15:55:13Z" message: Scaled up replica set ensemble-graph-687f8-5b9ff78587 from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:55:13Z" name: ensemble-graph-687f8.18b1511d399ca8a0 namespace: kserve-ci-e2e-test resourceVersion: "20518" uid: 16b194a4-13fd-4152-8d22-59394f5eac5e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:13Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-687f8 namespace: kserve-ci-e2e-test resourceVersion: "20531" uid: 33d08e1e-441b-4937-91f3-e3409990d6a1 kind: Event lastTimestamp: "2026-05-20T15:55:13Z" message: InferenceGraph [ensemble-graph-687f8] is Ready metadata: creationTimestamp: "2026-05-20T15:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:55:13Z" name: ensemble-graph-687f8.18b1511d403c9a9f namespace: kserve-ci-e2e-test resourceVersion: "20533" uid: 3bc3ba83-9890-4863-bbb9-bcd198a5fb78 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:45:14.707776Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-23504-predictor-6d476745d5-rhhv6 namespace: kserve-ci-e2e-test resourceVersion: "15792" uid: 45e8ec2e-cb37-46fd-9bca-db9bcd707e33 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-23504-predictor-6d476745d5-rhhv6 to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T15:45:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:45:14Z" name: error-404-isvc-23504-predictor-6d476745d5-rhhv6.18b15091bfe1e017 namespace: kserve-ci-e2e-test resourceVersion: "15799" uid: 6302a7d7-6576-4024-8ec3-5c54c9e59bc3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:15Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-23504-predictor-6d476745d5-rhhv6 namespace: kserve-ci-e2e-test resourceVersion: "15804" uid: 45e8ec2e-cb37-46fd-9bca-db9bcd707e33 kind: Event lastTimestamp: "2026-05-20T15:45:15Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:45:15Z" name: error-404-isvc-23504-predictor-6d476745d5-rhhv6.18b15091da7445fa namespace: kserve-ci-e2e-test resourceVersion: "15816" uid: fe6ae02f-e922-4902-a080-76d716e289ba reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-23504-predictor-6d476745d5-rhhv6 namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: 45e8ec2e-cb37-46fd-9bca-db9bcd707e33 kind: Event lastTimestamp: "2026-05-20T15:45:15Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-05-20T15:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:15Z" name: error-404-isvc-23504-predictor-6d476745d5-rhhv6.18b15091dc139f28 namespace: kserve-ci-e2e-test resourceVersion: "15818" uid: 2d5e0a81-95c4-4ff7-99bb-dbc20ae312b3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-23504-predictor-6d476745d5-rhhv6 namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: 45e8ec2e-cb37-46fd-9bca-db9bcd707e33 kind: Event lastTimestamp: "2026-05-20T15:45:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:15Z" name: error-404-isvc-23504-predictor-6d476745d5-rhhv6.18b15091df197c98 namespace: kserve-ci-e2e-test resourceVersion: "15821" uid: ec3b6934-df3d-41fd-999d-f4744c21a3b4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-23504-predictor-6d476745d5-rhhv6 namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: 45e8ec2e-cb37-46fd-9bca-db9bcd707e33 kind: Event lastTimestamp: "2026-05-20T15:45:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:15Z" name: error-404-isvc-23504-predictor-6d476745d5-rhhv6.18b15091df80e2a5 namespace: kserve-ci-e2e-test resourceVersion: "15822" uid: dcc524f0-02bc-40a3-93f7-02e3e2477673 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-23504-predictor-6d476745d5-rhhv6 namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: 45e8ec2e-cb37-46fd-9bca-db9bcd707e33 kind: Event lastTimestamp: "2026-05-20T15:45:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T15:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:15Z" name: error-404-isvc-23504-predictor-6d476745d5-rhhv6.18b15091df8c2823 namespace: kserve-ci-e2e-test resourceVersion: "15823" uid: 411eb6da-14ac-4611-b467-db7045481dbe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-23504-predictor-6d476745d5-rhhv6 namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: 45e8ec2e-cb37-46fd-9bca-db9bcd707e33 kind: Event lastTimestamp: "2026-05-20T15:45:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:15Z" name: error-404-isvc-23504-predictor-6d476745d5-rhhv6.18b15091e338d9fa namespace: kserve-ci-e2e-test resourceVersion: "15825" uid: 72dfdfab-6869-46c1-881a-3c1eb509241d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-23504-predictor-6d476745d5-rhhv6 namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: 45e8ec2e-cb37-46fd-9bca-db9bcd707e33 kind: Event lastTimestamp: "2026-05-20T15:45:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:15Z" name: error-404-isvc-23504-predictor-6d476745d5-rhhv6.18b15091e3af7eed namespace: kserve-ci-e2e-test resourceVersion: "15826" uid: ec4e3822-5c61-4980-a525-95e28e42551a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-20T15:45:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-23504-predictor-6d476745d5-rhhv6 namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: 45e8ec2e-cb37-46fd-9bca-db9bcd707e33 kind: Event lastTimestamp: "2026-05-20T15:54:32Z" message: 'Readiness probe failed: dial tcp 10.132.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:32Z" name: error-404-isvc-23504-predictor-6d476745d5-rhhv6.18b15092290621d4 namespace: kserve-ci-e2e-test resourceVersion: "20144" uid: f92d1b90-e20d-47b3-b1ef-35c9e0dd002d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-23504-predictor-6d476745d5-rhhv6 namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: 45e8ec2e-cb37-46fd-9bca-db9bcd707e33 kind: Event lastTimestamp: "2026-05-20T15:54:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T15:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:29Z" name: error-404-isvc-23504-predictor-6d476745d5-rhhv6.18b15112eb43777c namespace: kserve-ci-e2e-test resourceVersion: "20057" uid: 2c46e310-7110-4d11-8f18-6bb7c14fdd9b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-23504-predictor-6d476745d5-rhhv6 namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: 45e8ec2e-cb37-46fd-9bca-db9bcd707e33 kind: Event lastTimestamp: "2026-05-20T15:54:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:29Z" name: error-404-isvc-23504-predictor-6d476745d5-rhhv6.18b15112eb4a60f3 namespace: kserve-ci-e2e-test resourceVersion: "20059" uid: 35edcdb0-8d4a-4085-ab8c-16671ed466ae reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-23504-predictor-6d476745d5-rhhv6 namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: 45e8ec2e-cb37-46fd-9bca-db9bcd707e33 kind: Event lastTimestamp: "2026-05-20T15:54:32Z" message: 'Readiness probe failed: Get "https://10.132.0.43:8643/healthz": dial tcp 10.132.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:32Z" name: error-404-isvc-23504-predictor-6d476745d5-rhhv6.18b151139d68d2b6 namespace: kserve-ci-e2e-test resourceVersion: "20142" uid: ea2e24f4-a8af-4317-b5f3-6c38623cc043 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-23504-predictor-6d476745d5 namespace: kserve-ci-e2e-test resourceVersion: "15776" uid: acbd3fd7-1140-4f9f-b67e-5ee5a20f146e kind: Event lastTimestamp: "2026-05-20T15:45:14Z" message: 'Created pod: error-404-isvc-23504-predictor-6d476745d5-rhhv6' metadata: creationTimestamp: "2026-05-20T15:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:14Z" name: error-404-isvc-23504-predictor-6d476745d5.18b15091bf347aa1 namespace: kserve-ci-e2e-test resourceVersion: "15795" uid: f393543a-bcfd-41c1-a3e5-e1a259633dda reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-23504-predictor namespace: kserve-ci-e2e-test resourceVersion: "15770" uid: a5997442-05a2-481e-afd7-5c77b5508e9f kind: Event lastTimestamp: "2026-05-20T15:45:14Z" message: Scaled up replica set error-404-isvc-23504-predictor-6d476745d5 from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:14Z" name: error-404-isvc-23504-predictor.18b15091bcd36880 namespace: kserve-ci-e2e-test resourceVersion: "15780" uid: 53afdbcc-031d-433c-80e3-411447036db5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:45:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-23504-predictor namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 399b6df9-46da-4374-a56a-1946527bd649 kind: Event lastTimestamp: "2026-05-20T15:45:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:45:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:59Z" name: error-404-isvc-23504-predictor.18b150953b9a4964 namespace: kserve-ci-e2e-test resourceVersion: "16273" uid: 9204e5a5-ad2a-4477-b77d-e09ea41cdf61 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:45:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-23504-predictor namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 399b6df9-46da-4374-a56a-1946527bd649 kind: Event lastTimestamp: "2026-05-20T15:45:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:45:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:59Z" name: error-404-isvc-23504-predictor.18b150953c0e0ab8 namespace: kserve-ci-e2e-test resourceVersion: "16274" uid: 58bcd8f0-b067-4822-89ee-bc3570825c99 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:46:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-23504-predictor namespace: kserve-ci-e2e-test resourceVersion: "15984" uid: 399b6df9-46da-4374-a56a-1946527bd649 kind: Event lastTimestamp: "2026-05-20T15:46:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:46:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:46:29Z" name: error-404-isvc-23504-predictor.18b1509fb89507aa namespace: kserve-ci-e2e-test resourceVersion: "16557" uid: c11d8b8f-cc22-4609-81fb-eaa86494eab1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:46:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-23504-predictor namespace: kserve-ci-e2e-test resourceVersion: "15984" uid: 399b6df9-46da-4374-a56a-1946527bd649 kind: Event lastTimestamp: "2026-05-20T15:46:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:46:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:46:29Z" name: error-404-isvc-23504-predictor.18b1509fb9029449 namespace: kserve-ci-e2e-test resourceVersion: "16558" uid: 7b973869-4da2-489b-a2d4-6ad8853e5a04 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T15:45:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-23504 namespace: kserve-ci-e2e-test resourceVersion: "15750" uid: 05e47956-2376-4710-97aa-3c3257d7ebcf kind: Event lastTimestamp: "2026-05-20T15:46:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:46:02Z" name: error-404-isvc-23504.18b15091bcd883e6 namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: 467a311e-daac-4408-8d32-70c681818bd1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-23504 namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 05e47956-2376-4710-97aa-3c3257d7ebcf kind: Event lastTimestamp: "2026-05-20T15:46:02Z" message: InferenceService [error-404-isvc-23504] is Ready metadata: creationTimestamp: "2026-05-20T15:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:46:02Z" name: error-404-isvc-23504.18b1509ce4beeb13 namespace: kserve-ci-e2e-test resourceVersion: "16302" uid: 9457cf42-8fc8-4c0c-a30c-3de81fd16e48 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:55:24.075902Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3f928-predictor-787568c877-rvqzn namespace: kserve-ci-e2e-test resourceVersion: "20730" uid: 0b9ffefe-0aad-4ca0-9b52-59b073671f33 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3f928-predictor-787568c877-rvqzn to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T15:55:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:55:24Z" name: error-404-isvc-3f928-predictor-787568c877-rvqzn.18b1511fa10d8af8 namespace: kserve-ci-e2e-test resourceVersion: "20734" uid: eb0f9e21-c8c9-46df-85c4-b4acc9d435d3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:24Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3f928-predictor-787568c877-rvqzn namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: 0b9ffefe-0aad-4ca0-9b52-59b073671f33 kind: Event lastTimestamp: "2026-05-20T15:55:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-3f928-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T15:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:24Z" name: error-404-isvc-3f928-predictor-787568c877-rvqzn.18b1511fadffb490 namespace: kserve-ci-e2e-test resourceVersion: "20744" uid: d20c63d9-29ef-4758-9205-a5eca43a2042 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:25Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3f928-predictor-787568c877-rvqzn namespace: kserve-ci-e2e-test resourceVersion: "20740" uid: 0b9ffefe-0aad-4ca0-9b52-59b073671f33 kind: Event lastTimestamp: "2026-05-20T15:55:25Z" message: Add eth0 [10.132.0.52/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:55:25Z" name: error-404-isvc-3f928-predictor-787568c877-rvqzn.18b1511fdf8401d0 namespace: kserve-ci-e2e-test resourceVersion: "20758" uid: 5870f184-4358-4d24-8ecf-0e7fcb0d7e21 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3f928-predictor-787568c877-rvqzn namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: 0b9ffefe-0aad-4ca0-9b52-59b073671f33 kind: Event lastTimestamp: "2026-05-20T15:55:25Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-05-20T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:25Z" name: error-404-isvc-3f928-predictor-787568c877-rvqzn.18b1511fe0d73375 namespace: kserve-ci-e2e-test resourceVersion: "20760" uid: fab4cb3f-ceda-44d0-98f7-5cd24772da55 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3f928-predictor-787568c877-rvqzn namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: 0b9ffefe-0aad-4ca0-9b52-59b073671f33 kind: Event lastTimestamp: "2026-05-20T15:55:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:25Z" name: error-404-isvc-3f928-predictor-787568c877-rvqzn.18b1511fe4b9de73 namespace: kserve-ci-e2e-test resourceVersion: "20765" uid: 7c1360e6-1358-4636-ac88-e00cb09ac0be reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3f928-predictor-787568c877-rvqzn namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: 0b9ffefe-0aad-4ca0-9b52-59b073671f33 kind: Event lastTimestamp: "2026-05-20T15:55:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:25Z" name: error-404-isvc-3f928-predictor-787568c877-rvqzn.18b1511fe539d441 namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: ea2a436f-0251-4250-a7bd-1f3e9dc49354 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3f928-predictor-787568c877-rvqzn namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: 0b9ffefe-0aad-4ca0-9b52-59b073671f33 kind: Event lastTimestamp: "2026-05-20T15:55:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:25Z" name: error-404-isvc-3f928-predictor-787568c877-rvqzn.18b1511fe548e5a9 namespace: kserve-ci-e2e-test resourceVersion: "20768" uid: 9d68ed60-a997-4d6c-b279-f5d8e3fbd621 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3f928-predictor-787568c877-rvqzn namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: 0b9ffefe-0aad-4ca0-9b52-59b073671f33 kind: Event lastTimestamp: "2026-05-20T15:55:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:25Z" name: error-404-isvc-3f928-predictor-787568c877-rvqzn.18b1511fe8cfd3c3 namespace: kserve-ci-e2e-test resourceVersion: "20770" uid: f5c2b7f1-fff6-418f-80b6-07537aa9f683 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3f928-predictor-787568c877-rvqzn namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: 0b9ffefe-0aad-4ca0-9b52-59b073671f33 kind: Event lastTimestamp: "2026-05-20T15:55:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:25Z" name: error-404-isvc-3f928-predictor-787568c877-rvqzn.18b1511fe954b464 namespace: kserve-ci-e2e-test resourceVersion: "20772" uid: f4362ea2-6e20-4108-b7a4-dbdf0c7e2ac9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T15:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3f928-predictor-787568c877-rvqzn namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: 0b9ffefe-0aad-4ca0-9b52-59b073671f33 kind: Event lastTimestamp: "2026-05-20T15:56:02Z" message: 'Readiness probe failed: dial tcp 10.132.0.52:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:56:02Z" name: error-404-isvc-3f928-predictor-787568c877-rvqzn.18b151203d3d1f56 namespace: kserve-ci-e2e-test resourceVersion: "21297" uid: b444e5a6-f088-4ddf-9d9f-afa302a6e606 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3f928-predictor-787568c877-rvqzn namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: 0b9ffefe-0aad-4ca0-9b52-59b073671f33 kind: Event lastTimestamp: "2026-05-20T16:04:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T16:04:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:04:48Z" name: error-404-isvc-3f928-predictor-787568c877-rvqzn.18b151a326ae68ca namespace: kserve-ci-e2e-test resourceVersion: "25388" uid: 3b35a755-f708-44f5-9465-8391b66074f0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3f928-predictor-787568c877-rvqzn namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: 0b9ffefe-0aad-4ca0-9b52-59b073671f33 kind: Event lastTimestamp: "2026-05-20T16:04:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T16:04:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:04:48Z" name: error-404-isvc-3f928-predictor-787568c877-rvqzn.18b151a326afa778 namespace: kserve-ci-e2e-test resourceVersion: "25392" uid: af82818c-1c3e-40ec-a54d-1a597596f79b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3f928-predictor-787568c877 namespace: kserve-ci-e2e-test resourceVersion: "20721" uid: 1d493841-2af4-4ab9-8648-55e3e19cffd4 kind: Event lastTimestamp: "2026-05-20T15:55:24Z" message: 'Created pod: error-404-isvc-3f928-predictor-787568c877-rvqzn' metadata: creationTimestamp: "2026-05-20T15:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:55:24Z" name: error-404-isvc-3f928-predictor-787568c877.18b1511fa0a946c5 namespace: kserve-ci-e2e-test resourceVersion: "20731" uid: 2b57729e-0309-4fc4-914e-01ae8ce045df reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3f928-predictor namespace: kserve-ci-e2e-test resourceVersion: "20718" uid: bfedb135-7a7a-404c-9221-3b113fd53f7f kind: Event lastTimestamp: "2026-05-20T15:55:24Z" message: Scaled up replica set error-404-isvc-3f928-predictor-787568c877 from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:55:24Z" name: error-404-isvc-3f928-predictor.18b1511f9eca8e05 namespace: kserve-ci-e2e-test resourceVersion: "20724" uid: 35f2b4ad-7777-4411-b867-cbfd07c78047 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:55:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3f928-predictor namespace: kserve-ci-e2e-test resourceVersion: "20720" uid: 83d54dba-3b43-41cb-ab12-9f6e05bab91e kind: Event lastTimestamp: "2026-05-20T15:55:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:55:54Z" name: error-404-isvc-3f928-predictor.18b151231d67ff41 namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 24a372fc-3370-484a-8f16-fc49b4f4bd3a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:55:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3f928-predictor namespace: kserve-ci-e2e-test resourceVersion: "20720" uid: 83d54dba-3b43-41cb-ab12-9f6e05bab91e kind: Event lastTimestamp: "2026-05-20T15:55:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:55:54Z" name: error-404-isvc-3f928-predictor.18b151231dd2c864 namespace: kserve-ci-e2e-test resourceVersion: "21224" uid: cac3b43c-bfb6-44f0-8c1a-7815882b53ce reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:56:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3f928-predictor namespace: kserve-ci-e2e-test resourceVersion: "20955" uid: 83d54dba-3b43-41cb-ab12-9f6e05bab91e kind: Event lastTimestamp: "2026-05-20T15:56:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:56:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:56:24Z" name: error-404-isvc-3f928-predictor.18b1512a1c3e0ff2 namespace: kserve-ci-e2e-test resourceVersion: "21479" uid: accb88bb-f29c-4406-803b-622646ab91d5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:56:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3f928-predictor namespace: kserve-ci-e2e-test resourceVersion: "20955" uid: 83d54dba-3b43-41cb-ab12-9f6e05bab91e kind: Event lastTimestamp: "2026-05-20T15:56:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:56:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:56:24Z" name: error-404-isvc-3f928-predictor.18b1512a1cb85ce5 namespace: kserve-ci-e2e-test resourceVersion: "21480" uid: 1151fb12-aee6-48f9-bb36-6f60f69d7200 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T15:55:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3f928 namespace: kserve-ci-e2e-test resourceVersion: "20689" uid: 953e89ae-33c4-4381-a77f-ce4ffacfc995 kind: Event lastTimestamp: "2026-05-20T15:56:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:56:12Z" name: error-404-isvc-3f928.18b1511f9ea299db namespace: kserve-ci-e2e-test resourceVersion: "21393" uid: 6bb0aa37-f28e-444c-abb2-1e5aac524fee reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3f928 namespace: kserve-ci-e2e-test resourceVersion: "20689" uid: 953e89ae-33c4-4381-a77f-ce4ffacfc995 kind: Event lastTimestamp: "2026-05-20T15:55:24Z" message: 'Failed to update status for InferenceService "error-404-isvc-3f928": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3f928": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:55:24Z" name: error-404-isvc-3f928.18b1511f9f2bf4d5 namespace: kserve-ci-e2e-test resourceVersion: "20726" uid: e4bb4c68-1f23-48f2-ab49-772724c434a8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3f928 namespace: kserve-ci-e2e-test resourceVersion: "20689" uid: 953e89ae-33c4-4381-a77f-ce4ffacfc995 kind: Event lastTimestamp: "2026-05-20T15:55:24Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3f928": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:55:24Z" name: error-404-isvc-3f928.18b1511f9f2c64f6 namespace: kserve-ci-e2e-test resourceVersion: "20727" uid: f2b95b2a-426d-4e7c-8246-dc41f0fd50b0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:56:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3f928 namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: 953e89ae-33c4-4381-a77f-ce4ffacfc995 kind: Event lastTimestamp: "2026-05-20T15:56:12Z" message: InferenceService [error-404-isvc-3f928] is Ready metadata: creationTimestamp: "2026-05-20T15:56:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:56:12Z" name: error-404-isvc-3f928.18b1512af93138cf namespace: kserve-ci-e2e-test resourceVersion: "21392" uid: 109cc962-b5ed-4889-9581-9a66d9c5d5a6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:54:29.577867Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4ff76-predictor-658c9d7c5-bnlbp namespace: kserve-ci-e2e-test resourceVersion: "20078" uid: da04dcf0-5363-4b02-b5ba-8e39acd77721 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4ff76-predictor-658c9d7c5-bnlbp to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T15:54:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:54:29Z" name: error-404-isvc-4ff76-predictor-658c9d7c5-bnlbp.18b15112f0b77c4d namespace: kserve-ci-e2e-test resourceVersion: "20082" uid: 7f4bde4d-219b-4c85-abb1-b90e853a4c8c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:30Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4ff76-predictor-658c9d7c5-bnlbp namespace: kserve-ci-e2e-test resourceVersion: "20088" uid: da04dcf0-5363-4b02-b5ba-8e39acd77721 kind: Event lastTimestamp: "2026-05-20T15:54:30Z" message: Add eth0 [10.132.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:54:30Z" name: error-404-isvc-4ff76-predictor-658c9d7c5-bnlbp.18b151130b3d3c84 namespace: kserve-ci-e2e-test resourceVersion: "20101" uid: 833764d0-3cc0-427b-a978-a535cbd51800 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4ff76-predictor-658c9d7c5-bnlbp namespace: kserve-ci-e2e-test resourceVersion: "20079" uid: da04dcf0-5363-4b02-b5ba-8e39acd77721 kind: Event lastTimestamp: "2026-05-20T15:54:30Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-05-20T15:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:30Z" name: error-404-isvc-4ff76-predictor-658c9d7c5-bnlbp.18b151130ccea217 namespace: kserve-ci-e2e-test resourceVersion: "20103" uid: e60039e9-271e-4391-b349-8b75eed4bec1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4ff76-predictor-658c9d7c5-bnlbp namespace: kserve-ci-e2e-test resourceVersion: "20079" uid: da04dcf0-5363-4b02-b5ba-8e39acd77721 kind: Event lastTimestamp: "2026-05-20T15:54:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:30Z" name: error-404-isvc-4ff76-predictor-658c9d7c5-bnlbp.18b151130fedf956 namespace: kserve-ci-e2e-test resourceVersion: "20105" uid: d2ba2e95-2c6f-4c24-9ebe-016192e516ca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4ff76-predictor-658c9d7c5-bnlbp namespace: kserve-ci-e2e-test resourceVersion: "20079" uid: da04dcf0-5363-4b02-b5ba-8e39acd77721 kind: Event lastTimestamp: "2026-05-20T15:54:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:30Z" name: error-404-isvc-4ff76-predictor-658c9d7c5-bnlbp.18b15113106ac40b namespace: kserve-ci-e2e-test resourceVersion: "20106" uid: 8fb14f07-1647-4271-b5e2-4fbe0f471946 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4ff76-predictor-658c9d7c5-bnlbp namespace: kserve-ci-e2e-test resourceVersion: "20079" uid: da04dcf0-5363-4b02-b5ba-8e39acd77721 kind: Event lastTimestamp: "2026-05-20T15:54:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T15:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:30Z" name: error-404-isvc-4ff76-predictor-658c9d7c5-bnlbp.18b151131077c3ce namespace: kserve-ci-e2e-test resourceVersion: "20107" uid: adc76aad-f563-40ba-8556-fad8a087f973 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4ff76-predictor-658c9d7c5-bnlbp namespace: kserve-ci-e2e-test resourceVersion: "20079" uid: da04dcf0-5363-4b02-b5ba-8e39acd77721 kind: Event lastTimestamp: "2026-05-20T15:54:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:30Z" name: error-404-isvc-4ff76-predictor-658c9d7c5-bnlbp.18b1511313659774 namespace: kserve-ci-e2e-test resourceVersion: "20108" uid: 9b61c38e-7283-4358-865f-b61d3442b36e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4ff76-predictor-658c9d7c5-bnlbp namespace: kserve-ci-e2e-test resourceVersion: "20079" uid: da04dcf0-5363-4b02-b5ba-8e39acd77721 kind: Event lastTimestamp: "2026-05-20T15:54:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:30Z" name: error-404-isvc-4ff76-predictor-658c9d7c5-bnlbp.18b1511313e08a37 namespace: kserve-ci-e2e-test resourceVersion: "20109" uid: 8e6aecbc-ce19-4464-a1e0-6a97208a324a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-20T15:54:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4ff76-predictor-658c9d7c5-bnlbp namespace: kserve-ci-e2e-test resourceVersion: "20079" uid: da04dcf0-5363-4b02-b5ba-8e39acd77721 kind: Event lastTimestamp: "2026-05-20T15:55:17Z" message: 'Readiness probe failed: dial tcp 10.132.0.49:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:17Z" name: error-404-isvc-4ff76-predictor-658c9d7c5-bnlbp.18b151136225e8b8 namespace: kserve-ci-e2e-test resourceVersion: "20575" uid: ddb2370a-c964-4f9b-8992-844b6415782a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4ff76-predictor-658c9d7c5-bnlbp namespace: kserve-ci-e2e-test resourceVersion: "20079" uid: da04dcf0-5363-4b02-b5ba-8e39acd77721 kind: Event lastTimestamp: "2026-05-20T15:55:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T15:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:39Z" name: error-404-isvc-4ff76-predictor-658c9d7c5-bnlbp.18b1512343d32dfa namespace: kserve-ci-e2e-test resourceVersion: "21000" uid: f597b412-d803-40cb-a717-cff821204421 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4ff76-predictor-658c9d7c5-bnlbp namespace: kserve-ci-e2e-test resourceVersion: "20079" uid: da04dcf0-5363-4b02-b5ba-8e39acd77721 kind: Event lastTimestamp: "2026-05-20T15:55:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:39Z" name: error-404-isvc-4ff76-predictor-658c9d7c5-bnlbp.18b1512343d3971d namespace: kserve-ci-e2e-test resourceVersion: "21002" uid: 914ed27a-8be4-46d5-bbac-fc11dd420158 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4ff76-predictor-658c9d7c5-bnlbp namespace: kserve-ci-e2e-test resourceVersion: "20079" uid: da04dcf0-5363-4b02-b5ba-8e39acd77721 kind: Event lastTimestamp: "2026-05-20T15:55:42Z" message: 'Readiness probe failed: Get "https://10.132.0.49:8643/healthz": dial tcp 10.132.0.49:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:55:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:42Z" name: error-404-isvc-4ff76-predictor-658c9d7c5-bnlbp.18b15123ea52d227 namespace: kserve-ci-e2e-test resourceVersion: "21122" uid: 1c06357a-0134-4c38-96de-15f276d2551b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4ff76-predictor-658c9d7c5 namespace: kserve-ci-e2e-test resourceVersion: "20070" uid: 774b5126-9f8d-4226-b1f8-ef3fc5d016c1 kind: Event lastTimestamp: "2026-05-20T15:54:29Z" message: 'Created pod: error-404-isvc-4ff76-predictor-658c9d7c5-bnlbp' metadata: creationTimestamp: "2026-05-20T15:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:54:29Z" name: error-404-isvc-4ff76-predictor-658c9d7c5.18b15112f084a319 namespace: kserve-ci-e2e-test resourceVersion: "20080" uid: ec59269b-aa6f-4156-9797-92ef7f82c8d3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4ff76-predictor namespace: kserve-ci-e2e-test resourceVersion: "20067" uid: 19d56bf8-8e54-4f41-8082-f34e9598f341 kind: Event lastTimestamp: "2026-05-20T15:54:29Z" message: Scaled up replica set error-404-isvc-4ff76-predictor-658c9d7c5 from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:54:29Z" name: error-404-isvc-4ff76-predictor.18b15112ee934e16 namespace: kserve-ci-e2e-test resourceVersion: "20072" uid: 8f7cc10a-a48a-4c1c-b8fa-7386d287ba36 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:54:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4ff76-predictor namespace: kserve-ci-e2e-test resourceVersion: "20068" uid: b3f3720d-6728-4621-a1ee-fd3ffb5bd4b3 kind: Event lastTimestamp: "2026-05-20T15:54:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:54:59Z" name: error-404-isvc-4ff76-predictor.18b151166d05f5f7 namespace: kserve-ci-e2e-test resourceVersion: "20388" uid: 19229ea6-9957-44ff-bf00-6784b4748a27 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:54:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4ff76-predictor namespace: kserve-ci-e2e-test resourceVersion: "20068" uid: b3f3720d-6728-4621-a1ee-fd3ffb5bd4b3 kind: Event lastTimestamp: "2026-05-20T15:54:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:54:59Z" name: error-404-isvc-4ff76-predictor.18b151166d75f4c5 namespace: kserve-ci-e2e-test resourceVersion: "20389" uid: 131e6668-012e-4883-aea0-990b1faadc25 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:55:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4ff76-predictor namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: b3f3720d-6728-4621-a1ee-fd3ffb5bd4b3 kind: Event lastTimestamp: "2026-05-20T15:55:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:55:29Z" name: error-404-isvc-4ff76-predictor.18b1511d6b330f9f namespace: kserve-ci-e2e-test resourceVersion: "20862" uid: b98ae445-42a0-4960-a88f-50b9ca00aa3f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:55:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4ff76-predictor namespace: kserve-ci-e2e-test resourceVersion: "20255" uid: b3f3720d-6728-4621-a1ee-fd3ffb5bd4b3 kind: Event lastTimestamp: "2026-05-20T15:55:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:55:29Z" name: error-404-isvc-4ff76-predictor.18b1511d6baa597b namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: 431e6c04-6430-4c0e-9974-68cb08f36d8b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T15:54:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4ff76 namespace: kserve-ci-e2e-test resourceVersion: "20044" uid: 9a5de2b5-598b-42da-84a1-d5e13f9af131 kind: Event lastTimestamp: "2026-05-20T15:55:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:55:27Z" name: error-404-isvc-4ff76.18b15112ee415d95 namespace: kserve-ci-e2e-test resourceVersion: "20810" uid: 5c725dae-9e68-4605-b9dd-7a94226606e6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4ff76 namespace: kserve-ci-e2e-test resourceVersion: "20044" uid: 9a5de2b5-598b-42da-84a1-d5e13f9af131 kind: Event lastTimestamp: "2026-05-20T15:54:29Z" message: 'Failed to update status for InferenceService "error-404-isvc-4ff76": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4ff76": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:54:29Z" name: error-404-isvc-4ff76.18b15112eeb18c5b namespace: kserve-ci-e2e-test resourceVersion: "20074" uid: 5f226465-0e46-499a-9a7b-cdd87483ba7b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4ff76 namespace: kserve-ci-e2e-test resourceVersion: "20044" uid: 9a5de2b5-598b-42da-84a1-d5e13f9af131 kind: Event lastTimestamp: "2026-05-20T15:54:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4ff76": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:54:29Z" name: error-404-isvc-4ff76.18b15112eeb1f77e namespace: kserve-ci-e2e-test resourceVersion: "20075" uid: c3eed0f5-c26e-46a3-82e8-8b864521d833 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4ff76 namespace: kserve-ci-e2e-test resourceVersion: "20808" uid: 9a5de2b5-598b-42da-84a1-d5e13f9af131 kind: Event lastTimestamp: "2026-05-20T15:55:27Z" message: InferenceService [error-404-isvc-4ff76] is Ready metadata: creationTimestamp: "2026-05-20T15:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:55:27Z" name: error-404-isvc-4ff76.18b1512071d33aa7 namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: 440e8b76-e448-41e0-8c0e-b28b3730bec5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:43:04.733104Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-542af-predictor-5854dfc889-f25kz namespace: kserve-ci-e2e-test resourceVersion: "14207" uid: 957084cc-60ef-4b62-b1e7-57642a9adad7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-542af-predictor-5854dfc889-f25kz to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T15:43:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:43:04Z" name: error-404-isvc-542af-predictor-5854dfc889-f25kz.18b150737cc9c1c5 namespace: kserve-ci-e2e-test resourceVersion: "14212" uid: 56fad740-f8b7-4151-92d2-ac77df84aafa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:05Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-542af-predictor-5854dfc889-f25kz namespace: kserve-ci-e2e-test resourceVersion: "14219" uid: 957084cc-60ef-4b62-b1e7-57642a9adad7 kind: Event lastTimestamp: "2026-05-20T15:43:05Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:43:05Z" name: error-404-isvc-542af-predictor-5854dfc889-f25kz.18b15073979e470c namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: aa1d562a-ea0b-4f54-8839-3dbd6afc33cc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-542af-predictor-5854dfc889-f25kz namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: 957084cc-60ef-4b62-b1e7-57642a9adad7 kind: Event lastTimestamp: "2026-05-20T15:43:05Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1433" metadata: creationTimestamp: "2026-05-20T15:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:05Z" name: error-404-isvc-542af-predictor-5854dfc889-f25kz.18b150739926487d namespace: kserve-ci-e2e-test resourceVersion: "14247" uid: bc19afd4-4f10-48c5-a6d7-d87f298eaa5a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-542af-predictor-5854dfc889-f25kz namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: 957084cc-60ef-4b62-b1e7-57642a9adad7 kind: Event lastTimestamp: "2026-05-20T15:43:20Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1433" in 14.944s (14.944s including waiting). Image size: 1334723675 bytes.' metadata: creationTimestamp: "2026-05-20T15:43:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:20Z" name: error-404-isvc-542af-predictor-5854dfc889-f25kz.18b1507713eddab6 namespace: kserve-ci-e2e-test resourceVersion: "14437" uid: 9d336c27-212c-430d-b357-775966d42b9e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-542af-predictor-5854dfc889-f25kz namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: 957084cc-60ef-4b62-b1e7-57642a9adad7 kind: Event lastTimestamp: "2026-05-20T15:43:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:43:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:20Z" name: error-404-isvc-542af-predictor-5854dfc889-f25kz.18b150771d2cb582 namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: ed9fd2ae-c5cc-48b2-a171-f00b603700f4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-542af-predictor-5854dfc889-f25kz namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: 957084cc-60ef-4b62-b1e7-57642a9adad7 kind: Event lastTimestamp: "2026-05-20T15:43:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:43:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:20Z" name: error-404-isvc-542af-predictor-5854dfc889-f25kz.18b150771e040d4a namespace: kserve-ci-e2e-test resourceVersion: "14443" uid: 4773689a-121c-4cd0-887f-3a0f8390ddff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-542af-predictor-5854dfc889-f25kz namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: 957084cc-60ef-4b62-b1e7-57642a9adad7 kind: Event lastTimestamp: "2026-05-20T15:43:20Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-20T15:43:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:20Z" name: error-404-isvc-542af-predictor-5854dfc889-f25kz.18b150771e109218 namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: f65f02b7-cda5-4dcd-a5a6-12b0b0b6e609 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-542af-predictor-5854dfc889-f25kz namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: 957084cc-60ef-4b62-b1e7-57642a9adad7 kind: Event lastTimestamp: "2026-05-20T15:43:24Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.613s (4.613s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-20T15:43:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:24Z" name: error-404-isvc-542af-predictor-5854dfc889-f25kz.18b1507831134d3d namespace: kserve-ci-e2e-test resourceVersion: "14479" uid: ae3b094f-7d02-4072-85b2-a0ede4cb7c7d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-542af-predictor-5854dfc889-f25kz namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: 957084cc-60ef-4b62-b1e7-57642a9adad7 kind: Event lastTimestamp: "2026-05-20T15:43:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:25Z" name: error-404-isvc-542af-predictor-5854dfc889-f25kz.18b1507835aed6a2 namespace: kserve-ci-e2e-test resourceVersion: "14482" uid: 62d30d61-265a-48d6-97a8-1acb94aa3eae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-542af-predictor-5854dfc889-f25kz namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: 957084cc-60ef-4b62-b1e7-57642a9adad7 kind: Event lastTimestamp: "2026-05-20T15:43:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:25Z" name: error-404-isvc-542af-predictor-5854dfc889-f25kz.18b1507836a94ded namespace: kserve-ci-e2e-test resourceVersion: "14484" uid: a6b600ea-8c64-4924-b885-068bc2714082 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-20T15:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-542af-predictor-5854dfc889-f25kz namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: 957084cc-60ef-4b62-b1e7-57642a9adad7 kind: Event lastTimestamp: "2026-05-20T15:44:03Z" message: 'Readiness probe failed: dial tcp 10.132.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:03Z" name: error-404-isvc-542af-predictor-5854dfc889-f25kz.18b1507872dc6e25 namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: 95be5606-3bf1-46dc-bf70-6c0cc6c1a199 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-542af-predictor-5854dfc889-f25kz namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: 957084cc-60ef-4b62-b1e7-57642a9adad7 kind: Event lastTimestamp: "2026-05-20T15:44:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T15:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:38Z" name: error-404-isvc-542af-predictor-5854dfc889-f25kz.18b1508968a2e234 namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: d101e63b-0782-4bf8-8a5f-e9c7ca370fe6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-542af-predictor-5854dfc889-f25kz namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: 957084cc-60ef-4b62-b1e7-57642a9adad7 kind: Event lastTimestamp: "2026-05-20T15:44:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:38Z" name: error-404-isvc-542af-predictor-5854dfc889-f25kz.18b1508968a374e9 namespace: kserve-ci-e2e-test resourceVersion: "15273" uid: 8691a1af-2885-4004-ac6b-c64a7cb7c544 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-542af-predictor-5854dfc889 namespace: kserve-ci-e2e-test resourceVersion: "14194" uid: 731fdfc2-de33-4347-b983-4037148a93b1 kind: Event lastTimestamp: "2026-05-20T15:43:04Z" message: 'Created pod: error-404-isvc-542af-predictor-5854dfc889-f25kz' metadata: creationTimestamp: "2026-05-20T15:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:43:04Z" name: error-404-isvc-542af-predictor-5854dfc889.18b150737c3e9a7c namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 38ac8dd1-dadc-40b3-a91d-34a3c698db00 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-542af-predictor namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: cad302da-9a83-43b1-8afb-33796432096c kind: Event lastTimestamp: "2026-05-20T15:43:04Z" message: Scaled up replica set error-404-isvc-542af-predictor-5854dfc889 from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:43:04Z" name: error-404-isvc-542af-predictor.18b150737729cfe7 namespace: kserve-ci-e2e-test resourceVersion: "14197" uid: 144f445d-c2d8-478a-a4fe-ef32878213a3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:43:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-542af-predictor namespace: kserve-ci-e2e-test resourceVersion: "14195" uid: 87028606-254c-493c-b8c2-e2fac10d22d0 kind: Event lastTimestamp: "2026-05-20T15:44:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:04Z" name: error-404-isvc-542af-predictor.18b15076f62709cb namespace: kserve-ci-e2e-test resourceVersion: "14849" uid: b682ff5a-d054-4018-8671-96ea99475726 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:43:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-542af-predictor namespace: kserve-ci-e2e-test resourceVersion: "14195" uid: 87028606-254c-493c-b8c2-e2fac10d22d0 kind: Event lastTimestamp: "2026-05-20T15:44:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:04Z" name: error-404-isvc-542af-predictor.18b15076f69745b9 namespace: kserve-ci-e2e-test resourceVersion: "14850" uid: 328182e8-6929-42ed-9046-35950e44a4fe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:43:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-542af-predictor namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: 87028606-254c-493c-b8c2-e2fac10d22d0 kind: Event lastTimestamp: "2026-05-20T15:44:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:34Z" name: error-404-isvc-542af-predictor.18b1507df4a4d828 namespace: kserve-ci-e2e-test resourceVersion: "15147" uid: 86b33c1d-1292-427a-aa96-b4d345f6b504 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:43:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-542af-predictor namespace: kserve-ci-e2e-test resourceVersion: "14430" uid: 87028606-254c-493c-b8c2-e2fac10d22d0 kind: Event lastTimestamp: "2026-05-20T15:44:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:34Z" name: error-404-isvc-542af-predictor.18b1507df5c39301 namespace: kserve-ci-e2e-test resourceVersion: "15148" uid: 738b78b9-8906-41c7-92cc-032f43e94b4a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T15:43:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-542af namespace: kserve-ci-e2e-test resourceVersion: "14177" uid: 4f5ce7b0-456a-494d-a5b6-f11b7d463668 kind: Event lastTimestamp: "2026-05-20T15:44:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:44:13Z" name: error-404-isvc-542af.18b15073775c6537 namespace: kserve-ci-e2e-test resourceVersion: "14930" uid: a50b0893-8a4a-4e3c-97da-d5c73d32f420 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-542af namespace: kserve-ci-e2e-test resourceVersion: "14926" uid: 4f5ce7b0-456a-494d-a5b6-f11b7d463668 kind: Event lastTimestamp: "2026-05-20T15:44:13Z" message: InferenceService [error-404-isvc-542af] is Ready metadata: creationTimestamp: "2026-05-20T15:44:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:44:13Z" name: error-404-isvc-542af.18b1508370ba94a9 namespace: kserve-ci-e2e-test resourceVersion: "14927" uid: 813bb457-9f8b-4d0f-bbeb-ce525996cf70 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T16:06:09.403150Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-62c44-predictor-76f5b9974c-6fr6t namespace: kserve-ci-e2e-test resourceVersion: "26431" uid: 92995157-1adc-4560-ae14-06cd7783fa6c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-62c44-predictor-76f5b9974c-6fr6t to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T16:06:09Z" name: error-404-isvc-62c44-predictor-76f5b9974c-6fr6t.18b151b5e18dd9b9 namespace: kserve-ci-e2e-test resourceVersion: "26435" uid: 5847395d-fa61-4e16-af72-113de62d03a9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-62c44-predictor-76f5b9974c-6fr6t namespace: kserve-ci-e2e-test resourceVersion: "26441" uid: 92995157-1adc-4560-ae14-06cd7783fa6c kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: Add eth0 [10.132.0.62/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T16:06:09Z" name: error-404-isvc-62c44-predictor-76f5b9974c-6fr6t.18b151b5fc25c182 namespace: kserve-ci-e2e-test resourceVersion: "26457" uid: a30ca794-d292-477b-8ec0-ed7f2786b70a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-62c44-predictor-76f5b9974c-6fr6t namespace: kserve-ci-e2e-test resourceVersion: "26432" uid: 92995157-1adc-4560-ae14-06cd7783fa6c kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:09Z" name: error-404-isvc-62c44-predictor-76f5b9974c-6fr6t.18b151b5fdbe6aa7 namespace: kserve-ci-e2e-test resourceVersion: "26459" uid: b05b657f-95ae-49ad-b5fe-9cb884e962f1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-62c44-predictor-76f5b9974c-6fr6t namespace: kserve-ci-e2e-test resourceVersion: "26432" uid: 92995157-1adc-4560-ae14-06cd7783fa6c kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:09Z" name: error-404-isvc-62c44-predictor-76f5b9974c-6fr6t.18b151b600b545c8 namespace: kserve-ci-e2e-test resourceVersion: "26462" uid: 84b528d6-fb6e-4c3f-83d7-cd94533bb3d1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-62c44-predictor-76f5b9974c-6fr6t namespace: kserve-ci-e2e-test resourceVersion: "26432" uid: 92995157-1adc-4560-ae14-06cd7783fa6c kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:09Z" name: error-404-isvc-62c44-predictor-76f5b9974c-6fr6t.18b151b6012ee7c5 namespace: kserve-ci-e2e-test resourceVersion: "26463" uid: 088bce96-8e1a-4040-af82-170691b16448 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-62c44-predictor-76f5b9974c-6fr6t namespace: kserve-ci-e2e-test resourceVersion: "26432" uid: 92995157-1adc-4560-ae14-06cd7783fa6c kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:09Z" name: error-404-isvc-62c44-predictor-76f5b9974c-6fr6t.18b151b6013c538d namespace: kserve-ci-e2e-test resourceVersion: "26464" uid: c06850f3-0973-4da0-8e30-afb8508cc682 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-62c44-predictor-76f5b9974c-6fr6t namespace: kserve-ci-e2e-test resourceVersion: "26432" uid: 92995157-1adc-4560-ae14-06cd7783fa6c kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:09Z" name: error-404-isvc-62c44-predictor-76f5b9974c-6fr6t.18b151b6040b2933 namespace: kserve-ci-e2e-test resourceVersion: "26465" uid: 4991c49c-9af2-453a-abf5-c8b0631bac28 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-62c44-predictor-76f5b9974c-6fr6t namespace: kserve-ci-e2e-test resourceVersion: "26432" uid: 92995157-1adc-4560-ae14-06cd7783fa6c kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:09Z" name: error-404-isvc-62c44-predictor-76f5b9974c-6fr6t.18b151b6048cc6df namespace: kserve-ci-e2e-test resourceVersion: "26466" uid: 7946fb01-8f90-42e4-8d9c-268805870685 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-20T16:06:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-62c44-predictor-76f5b9974c-6fr6t namespace: kserve-ci-e2e-test resourceVersion: "26432" uid: 92995157-1adc-4560-ae14-06cd7783fa6c kind: Event lastTimestamp: "2026-05-20T16:15:26Z" message: 'Readiness probe failed: dial tcp 10.132.0.62:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T16:06:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:15:26Z" name: error-404-isvc-62c44-predictor-76f5b9974c-6fr6t.18b151b6057784a5 namespace: kserve-ci-e2e-test resourceVersion: "30514" uid: 04a08ffb-f0ca-4b42-8e05-6a5bdf3ab0f0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:15:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-62c44-predictor-76f5b9974c-6fr6t namespace: kserve-ci-e2e-test resourceVersion: "26432" uid: 92995157-1adc-4560-ae14-06cd7783fa6c kind: Event lastTimestamp: "2026-05-20T16:15:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T16:15:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:15:24Z" name: error-404-isvc-62c44-predictor-76f5b9974c-6fr6t.18b15237135ab541 namespace: kserve-ci-e2e-test resourceVersion: "30498" uid: c01142a6-67f2-46cc-ad35-05e26e6fdcef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:15:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-62c44-predictor-76f5b9974c-6fr6t namespace: kserve-ci-e2e-test resourceVersion: "26432" uid: 92995157-1adc-4560-ae14-06cd7783fa6c kind: Event lastTimestamp: "2026-05-20T16:15:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T16:15:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:15:24Z" name: error-404-isvc-62c44-predictor-76f5b9974c-6fr6t.18b15237135beedf namespace: kserve-ci-e2e-test resourceVersion: "30499" uid: 20e4ed0b-09d6-4797-bdd9-a59df5955d78 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:15:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-62c44-predictor-76f5b9974c-6fr6t namespace: kserve-ci-e2e-test resourceVersion: "26432" uid: 92995157-1adc-4560-ae14-06cd7783fa6c kind: Event lastTimestamp: "2026-05-20T16:15:26Z" message: 'Readiness probe failed: Get "https://10.132.0.62:8643/healthz": dial tcp 10.132.0.62:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T16:15:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:15:26Z" name: error-404-isvc-62c44-predictor-76f5b9974c-6fr6t.18b152377a10d918 namespace: kserve-ci-e2e-test resourceVersion: "30511" uid: c115bba8-40d0-4560-a381-57e1ecda0646 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-62c44-predictor-76f5b9974c namespace: kserve-ci-e2e-test resourceVersion: "26424" uid: 585fe412-89f9-457f-967f-bcfa74b5c774 kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: 'Created pod: error-404-isvc-62c44-predictor-76f5b9974c-6fr6t' metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:06:09Z" name: error-404-isvc-62c44-predictor-76f5b9974c.18b151b5e135eaf9 namespace: kserve-ci-e2e-test resourceVersion: "26433" uid: 75ee0442-96cb-4101-9ee4-0a8ac7602b06 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-62c44-predictor namespace: kserve-ci-e2e-test resourceVersion: "26422" uid: 4a9ef38c-4c98-4f2a-a358-61f3e7bee2b4 kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: Scaled up replica set error-404-isvc-62c44-predictor-76f5b9974c from 0 to 1 metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:06:09Z" name: error-404-isvc-62c44-predictor.18b151b5df7db87a namespace: kserve-ci-e2e-test resourceVersion: "26426" uid: 688bf187-0250-487a-afed-50d565bc42a9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T16:06:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-62c44-predictor namespace: kserve-ci-e2e-test resourceVersion: "26423" uid: 1fa1cb6b-2936-4144-b64a-4c42df46a43a kind: Event lastTimestamp: "2026-05-20T16:07:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T16:06:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:07:24Z" name: error-404-isvc-62c44-predictor.18b151b95ea630ce namespace: kserve-ci-e2e-test resourceVersion: "27159" uid: db48e997-5ed5-4f09-8237-d7591bda7d13 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T16:06:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-62c44-predictor namespace: kserve-ci-e2e-test resourceVersion: "26423" uid: 1fa1cb6b-2936-4144-b64a-4c42df46a43a kind: Event lastTimestamp: "2026-05-20T16:07:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T16:06:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:07:24Z" name: error-404-isvc-62c44-predictor.18b151b95f089da9 namespace: kserve-ci-e2e-test resourceVersion: "27160" uid: 079ad606-50e3-4370-8b20-f66f338c2af1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-62c44-predictor namespace: kserve-ci-e2e-test resourceVersion: "26646" uid: 1fa1cb6b-2936-4144-b64a-4c42df46a43a kind: Event lastTimestamp: "2026-05-20T16:06:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T16:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:06:39Z" name: error-404-isvc-62c44-predictor.18b151bcddcde4ea namespace: kserve-ci-e2e-test resourceVersion: "26767" uid: 0e7cc8a7-e484-482d-8e08-1a6fe08f4888 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-62c44-predictor namespace: kserve-ci-e2e-test resourceVersion: "26646" uid: 1fa1cb6b-2936-4144-b64a-4c42df46a43a kind: Event lastTimestamp: "2026-05-20T16:06:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T16:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:06:39Z" name: error-404-isvc-62c44-predictor.18b151bcde234c6e namespace: kserve-ci-e2e-test resourceVersion: "26769" uid: eca9adf9-cc21-48cc-b874-5b39b318aa8c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-62c44 namespace: kserve-ci-e2e-test resourceVersion: "26392" uid: f1e6a3ef-e1a7-49d9-9d25-a210894ab4b3 kind: Event lastTimestamp: "2026-05-20T16:06:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:06:56Z" name: error-404-isvc-62c44.18b151b5df5546bd namespace: kserve-ci-e2e-test resourceVersion: "26912" uid: 809a3df0-44f2-4727-a89d-f23e35e201d2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-62c44 namespace: kserve-ci-e2e-test resourceVersion: "26392" uid: f1e6a3ef-e1a7-49d9-9d25-a210894ab4b3 kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: 'Failed to update status for InferenceService "error-404-isvc-62c44": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-62c44": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:06:09Z" name: error-404-isvc-62c44.18b151b5dfb57c7e namespace: kserve-ci-e2e-test resourceVersion: "26428" uid: 094efce0-7a62-47d6-9ac2-ac3865941450 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-62c44 namespace: kserve-ci-e2e-test resourceVersion: "26392" uid: f1e6a3ef-e1a7-49d9-9d25-a210894ab4b3 kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-62c44": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:06:09Z" name: error-404-isvc-62c44.18b151b5dfb5df6d namespace: kserve-ci-e2e-test resourceVersion: "26429" uid: 59fd74fa-0eb8-4fd4-9d56-7a39a4ddb557 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-62c44 namespace: kserve-ci-e2e-test resourceVersion: "26907" uid: f1e6a3ef-e1a7-49d9-9d25-a210894ab4b3 kind: Event lastTimestamp: "2026-05-20T16:06:56Z" message: InferenceService [error-404-isvc-62c44] is Ready metadata: creationTimestamp: "2026-05-20T16:06:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:06:56Z" name: error-404-isvc-62c44.18b151c0c27050dd namespace: kserve-ci-e2e-test resourceVersion: "26908" uid: 44fcca46-8352-4663-9991-7a72b8f4f947 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:54:03.822038Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-687f8-predictor-d97cfdf7f-2xmtw namespace: kserve-ci-e2e-test resourceVersion: "19739" uid: 16b42b9f-fa74-4e57-8b69-f166541af5bd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-687f8-predictor-d97cfdf7f-2xmtw to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T15:54:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:54:03Z" name: error-404-isvc-687f8-predictor-d97cfdf7f-2xmtw.18b1510cf18cc118 namespace: kserve-ci-e2e-test resourceVersion: "19743" uid: 17476ada-ee7d-4a53-b07a-74bace51680d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:04Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-687f8-predictor-d97cfdf7f-2xmtw namespace: kserve-ci-e2e-test resourceVersion: "19747" uid: 16b42b9f-fa74-4e57-8b69-f166541af5bd kind: Event lastTimestamp: "2026-05-20T15:54:04Z" message: Add eth0 [10.132.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:54:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:54:04Z" name: error-404-isvc-687f8-predictor-d97cfdf7f-2xmtw.18b1510d0bbde429 namespace: kserve-ci-e2e-test resourceVersion: "19759" uid: c20a4aec-255d-4eb3-a747-ea46a5a904dc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-687f8-predictor-d97cfdf7f-2xmtw namespace: kserve-ci-e2e-test resourceVersion: "19740" uid: 16b42b9f-fa74-4e57-8b69-f166541af5bd kind: Event lastTimestamp: "2026-05-20T15:54:04Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-05-20T15:54:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:04Z" name: error-404-isvc-687f8-predictor-d97cfdf7f-2xmtw.18b1510d0d2ad173 namespace: kserve-ci-e2e-test resourceVersion: "19761" uid: 47372aea-203f-4851-8ee4-8b686dceb41d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-687f8-predictor-d97cfdf7f-2xmtw namespace: kserve-ci-e2e-test resourceVersion: "19740" uid: 16b42b9f-fa74-4e57-8b69-f166541af5bd kind: Event lastTimestamp: "2026-05-20T15:54:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:54:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:04Z" name: error-404-isvc-687f8-predictor-d97cfdf7f-2xmtw.18b1510d1037b5f3 namespace: kserve-ci-e2e-test resourceVersion: "19762" uid: 1f318183-ee89-48e1-869c-d9512d1a6cf6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-687f8-predictor-d97cfdf7f-2xmtw namespace: kserve-ci-e2e-test resourceVersion: "19740" uid: 16b42b9f-fa74-4e57-8b69-f166541af5bd kind: Event lastTimestamp: "2026-05-20T15:54:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:54:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:04Z" name: error-404-isvc-687f8-predictor-d97cfdf7f-2xmtw.18b1510d10cf2d60 namespace: kserve-ci-e2e-test resourceVersion: "19763" uid: f0681683-7e0f-4ac5-96ce-392f9552df85 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-687f8-predictor-d97cfdf7f-2xmtw namespace: kserve-ci-e2e-test resourceVersion: "19740" uid: 16b42b9f-fa74-4e57-8b69-f166541af5bd kind: Event lastTimestamp: "2026-05-20T15:54:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T15:54:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:04Z" name: error-404-isvc-687f8-predictor-d97cfdf7f-2xmtw.18b1510d10dc2323 namespace: kserve-ci-e2e-test resourceVersion: "19764" uid: 91b5dd14-dc4f-48c0-85f2-7ad747c9c6ca reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-687f8-predictor-d97cfdf7f-2xmtw namespace: kserve-ci-e2e-test resourceVersion: "19740" uid: 16b42b9f-fa74-4e57-8b69-f166541af5bd kind: Event lastTimestamp: "2026-05-20T15:54:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:54:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:04Z" name: error-404-isvc-687f8-predictor-d97cfdf7f-2xmtw.18b1510d13fb8df4 namespace: kserve-ci-e2e-test resourceVersion: "19765" uid: 796950a6-bffa-4f0a-9355-21bc2a5983dc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-687f8-predictor-d97cfdf7f-2xmtw namespace: kserve-ci-e2e-test resourceVersion: "19740" uid: 16b42b9f-fa74-4e57-8b69-f166541af5bd kind: Event lastTimestamp: "2026-05-20T15:54:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:54:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:04Z" name: error-404-isvc-687f8-predictor-d97cfdf7f-2xmtw.18b1510d14730191 namespace: kserve-ci-e2e-test resourceVersion: "19766" uid: f3633741-ad3b-4c2f-be23-2eb38ca41dfb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T15:54:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-687f8-predictor-d97cfdf7f-2xmtw namespace: kserve-ci-e2e-test resourceVersion: "19740" uid: 16b42b9f-fa74-4e57-8b69-f166541af5bd kind: Event lastTimestamp: "2026-05-20T15:54:41Z" message: 'Readiness probe failed: dial tcp 10.132.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:54:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:41Z" name: error-404-isvc-687f8-predictor-d97cfdf7f-2xmtw.18b1510d4dbddd7e namespace: kserve-ci-e2e-test resourceVersion: "20228" uid: 41bd9268-3a74-43a6-a401-d4d6965b7ffe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-687f8-predictor-d97cfdf7f-2xmtw namespace: kserve-ci-e2e-test resourceVersion: "19740" uid: 16b42b9f-fa74-4e57-8b69-f166541af5bd kind: Event lastTimestamp: "2026-05-20T15:55:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T15:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:24Z" name: error-404-isvc-687f8-predictor-d97cfdf7f-2xmtw.18b1511f9dff9eeb namespace: kserve-ci-e2e-test resourceVersion: "20719" uid: f4112448-5d7e-4b70-8d05-f0f6d1d48699 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-687f8-predictor-d97cfdf7f-2xmtw namespace: kserve-ci-e2e-test resourceVersion: "19740" uid: 16b42b9f-fa74-4e57-8b69-f166541af5bd kind: Event lastTimestamp: "2026-05-20T15:55:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:24Z" name: error-404-isvc-687f8-predictor-d97cfdf7f-2xmtw.18b1511f9e00e035 namespace: kserve-ci-e2e-test resourceVersion: "20722" uid: 43794c55-9dd8-42dd-b07f-f6eaeee7c4b2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-687f8-predictor-d97cfdf7f-2xmtw namespace: kserve-ci-e2e-test resourceVersion: "19740" uid: 16b42b9f-fa74-4e57-8b69-f166541af5bd kind: Event lastTimestamp: "2026-05-20T15:55:26Z" message: 'Readiness probe failed: Get "https://10.132.0.47:8643/healthz": dial tcp 10.132.0.47:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:26Z" name: error-404-isvc-687f8-predictor-d97cfdf7f-2xmtw.18b151202a00749f namespace: kserve-ci-e2e-test resourceVersion: "20792" uid: 8149d075-c0c8-4746-b536-0721fd96abe9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-687f8-predictor-d97cfdf7f namespace: kserve-ci-e2e-test resourceVersion: "19728" uid: a524e9f8-1a8d-433a-80ee-c9c8fab087a3 kind: Event lastTimestamp: "2026-05-20T15:54:03Z" message: 'Created pod: error-404-isvc-687f8-predictor-d97cfdf7f-2xmtw' metadata: creationTimestamp: "2026-05-20T15:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:54:03Z" name: error-404-isvc-687f8-predictor-d97cfdf7f.18b1510cf13c05f1 namespace: kserve-ci-e2e-test resourceVersion: "19741" uid: 8a433190-0eed-4939-8a08-fb1fce74b770 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-687f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19722" uid: 98aa8b13-9b66-4650-a23e-273f28123cc4 kind: Event lastTimestamp: "2026-05-20T15:54:03Z" message: Scaled up replica set error-404-isvc-687f8-predictor-d97cfdf7f from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:54:03Z" name: error-404-isvc-687f8-predictor.18b1510cef536e59 namespace: kserve-ci-e2e-test resourceVersion: "19731" uid: 8a441fe7-870d-423c-bd60-8d53c915d5cb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:54:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-687f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19727" uid: e77f059f-5c12-4fe8-b331-18d8d1138d9d kind: Event lastTimestamp: "2026-05-20T15:54:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:54:33Z" name: error-404-isvc-687f8-predictor.18b151106e2eaa94 namespace: kserve-ci-e2e-test resourceVersion: "20162" uid: 04e8f17c-15cf-45ef-988f-2f46b7018966 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:54:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-687f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19727" uid: e77f059f-5c12-4fe8-b331-18d8d1138d9d kind: Event lastTimestamp: "2026-05-20T15:54:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:54:33Z" name: error-404-isvc-687f8-predictor.18b151106ea020de namespace: kserve-ci-e2e-test resourceVersion: "20163" uid: a83cf95b-639c-442a-aa7f-150a39f4bfae reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:54:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-687f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19900" uid: e77f059f-5c12-4fe8-b331-18d8d1138d9d kind: Event lastTimestamp: "2026-05-20T15:55:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:55:18Z" name: error-404-isvc-687f8-predictor.18b151176c70a3bd namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: a8b5b8f3-c47e-4054-a6f6-b327fce2f181 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:54:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-687f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19900" uid: e77f059f-5c12-4fe8-b331-18d8d1138d9d kind: Event lastTimestamp: "2026-05-20T15:55:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:55:18Z" name: error-404-isvc-687f8-predictor.18b151176ce848c6 namespace: kserve-ci-e2e-test resourceVersion: "20596" uid: 251f9e3d-9806-407e-9bf7-bbb1162739f9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T15:54:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-687f8 namespace: kserve-ci-e2e-test resourceVersion: "19696" uid: c00b4396-cf93-48ab-8f19-92a8c6817a51 kind: Event lastTimestamp: "2026-05-20T15:54:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:54:51Z" name: error-404-isvc-687f8.18b1510cef63660b namespace: kserve-ci-e2e-test resourceVersion: "20324" uid: cb7895e1-6654-47cb-bda7-d8b4ed14cbe1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-687f8 namespace: kserve-ci-e2e-test resourceVersion: "19696" uid: c00b4396-cf93-48ab-8f19-92a8c6817a51 kind: Event lastTimestamp: "2026-05-20T15:54:03Z" message: 'Failed to update status for InferenceService "error-404-isvc-687f8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-687f8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:54:03Z" name: error-404-isvc-687f8.18b1510cf00a55f4 namespace: kserve-ci-e2e-test resourceVersion: "19736" uid: 0d7cfde4-af70-44ab-ad8c-68109bc1589b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-687f8 namespace: kserve-ci-e2e-test resourceVersion: "19696" uid: c00b4396-cf93-48ab-8f19-92a8c6817a51 kind: Event lastTimestamp: "2026-05-20T15:54:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-687f8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:54:03Z" name: error-404-isvc-687f8.18b1510cf00ac276 namespace: kserve-ci-e2e-test resourceVersion: "19737" uid: 3ea230e4-c787-4441-822c-e76155531689 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-687f8 namespace: kserve-ci-e2e-test resourceVersion: "20322" uid: c00b4396-cf93-48ab-8f19-92a8c6817a51 kind: Event lastTimestamp: "2026-05-20T15:54:51Z" message: InferenceService [error-404-isvc-687f8] is Ready metadata: creationTimestamp: "2026-05-20T15:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:54:51Z" name: error-404-isvc-687f8.18b151180940667a namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: fd8077d0-b9be-422e-9c74-d2ad2047a406 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:55:39.918892Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-80b6e-predictor-7f59bbd875-8td9x namespace: kserve-ci-e2e-test resourceVersion: "21056" uid: 0a8d08a2-7d2b-4b3f-9064-8c59b18cab79 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-80b6e-predictor-7f59bbd875-8td9x to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T15:55:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:55:39Z" name: error-404-isvc-80b6e-predictor-7f59bbd875-8td9x.18b15123515e6ee4 namespace: kserve-ci-e2e-test resourceVersion: "21061" uid: 2f8f803e-bed2-4e8e-8014-0dfdf46484fa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:40Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-80b6e-predictor-7f59bbd875-8td9x namespace: kserve-ci-e2e-test resourceVersion: "21068" uid: 0a8d08a2-7d2b-4b3f-9064-8c59b18cab79 kind: Event lastTimestamp: "2026-05-20T15:55:40Z" message: Add eth0 [10.132.0.55/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:55:40Z" name: error-404-isvc-80b6e-predictor-7f59bbd875-8td9x.18b151236b9b81a0 namespace: kserve-ci-e2e-test resourceVersion: "21082" uid: 20b761e7-1ad6-4066-94a6-ffa0e5d07df5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-80b6e-predictor-7f59bbd875-8td9x namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: 0a8d08a2-7d2b-4b3f-9064-8c59b18cab79 kind: Event lastTimestamp: "2026-05-20T15:55:40Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-05-20T15:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:40Z" name: error-404-isvc-80b6e-predictor-7f59bbd875-8td9x.18b151236d307b95 namespace: kserve-ci-e2e-test resourceVersion: "21084" uid: 411c1d47-bbef-4669-a4d1-c68cf9ae24bf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-80b6e-predictor-7f59bbd875-8td9x namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: 0a8d08a2-7d2b-4b3f-9064-8c59b18cab79 kind: Event lastTimestamp: "2026-05-20T15:55:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:40Z" name: error-404-isvc-80b6e-predictor-7f59bbd875-8td9x.18b151236f54d630 namespace: kserve-ci-e2e-test resourceVersion: "21087" uid: c4272dd0-4b76-4581-b390-e49bbe51be1f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-80b6e-predictor-7f59bbd875-8td9x namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: 0a8d08a2-7d2b-4b3f-9064-8c59b18cab79 kind: Event lastTimestamp: "2026-05-20T15:55:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:40Z" name: error-404-isvc-80b6e-predictor-7f59bbd875-8td9x.18b151236fd87138 namespace: kserve-ci-e2e-test resourceVersion: "21088" uid: 2fed7471-13a9-4115-941e-bd40dd48a290 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-80b6e-predictor-7f59bbd875-8td9x namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: 0a8d08a2-7d2b-4b3f-9064-8c59b18cab79 kind: Event lastTimestamp: "2026-05-20T15:55:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T15:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:40Z" name: error-404-isvc-80b6e-predictor-7f59bbd875-8td9x.18b151236fe3633e namespace: kserve-ci-e2e-test resourceVersion: "21089" uid: 97c76337-8e20-4ef4-bfb7-e3de1523b407 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-80b6e-predictor-7f59bbd875-8td9x namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: 0a8d08a2-7d2b-4b3f-9064-8c59b18cab79 kind: Event lastTimestamp: "2026-05-20T15:55:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:40Z" name: error-404-isvc-80b6e-predictor-7f59bbd875-8td9x.18b1512373f93ff3 namespace: kserve-ci-e2e-test resourceVersion: "21092" uid: 510fd234-5f25-42e6-8515-b14fe90e80fb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-80b6e-predictor-7f59bbd875-8td9x namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: 0a8d08a2-7d2b-4b3f-9064-8c59b18cab79 kind: Event lastTimestamp: "2026-05-20T15:55:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:40Z" name: error-404-isvc-80b6e-predictor-7f59bbd875-8td9x.18b15123746bef49 namespace: kserve-ci-e2e-test resourceVersion: "21093" uid: 0fd7edb8-1c3b-4f6b-b0de-599bfc211aa0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-20T15:55:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-80b6e-predictor-7f59bbd875-8td9x namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: 0a8d08a2-7d2b-4b3f-9064-8c59b18cab79 kind: Event lastTimestamp: "2026-05-20T16:05:07Z" message: 'Readiness probe failed: dial tcp 10.132.0.55:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:55:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:07Z" name: error-404-isvc-80b6e-predictor-7f59bbd875-8td9x.18b15123bf32dd26 namespace: kserve-ci-e2e-test resourceVersion: "25754" uid: 85ad8f49-a95a-4cdf-a529-8a598bb2afac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-80b6e-predictor-7f59bbd875-8td9x namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: 0a8d08a2-7d2b-4b3f-9064-8c59b18cab79 kind: Event lastTimestamp: "2026-05-20T16:05:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T16:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:04Z" name: error-404-isvc-80b6e-predictor-7f59bbd875-8td9x.18b151a6ce198060 namespace: kserve-ci-e2e-test resourceVersion: "25621" uid: d1bf68cd-7d3d-487d-9964-39794b99350a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-80b6e-predictor-7f59bbd875-8td9x namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: 0a8d08a2-7d2b-4b3f-9064-8c59b18cab79 kind: Event lastTimestamp: "2026-05-20T16:05:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T16:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:04Z" name: error-404-isvc-80b6e-predictor-7f59bbd875-8td9x.18b151a6ce1a0f84 namespace: kserve-ci-e2e-test resourceVersion: "25623" uid: b7f32f01-b5de-486a-a6e5-65c9f771e0b5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-80b6e-predictor-7f59bbd875-8td9x namespace: kserve-ci-e2e-test resourceVersion: "21059" uid: 0a8d08a2-7d2b-4b3f-9064-8c59b18cab79 kind: Event lastTimestamp: "2026-05-20T16:05:07Z" message: 'Readiness probe failed: Get "https://10.132.0.55:8643/healthz": dial tcp 10.132.0.55:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T16:05:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:07Z" name: error-404-isvc-80b6e-predictor-7f59bbd875-8td9x.18b151a787c7ce87 namespace: kserve-ci-e2e-test resourceVersion: "25752" uid: 573dc9a3-3420-475d-97b1-d2bb5d3c036f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-80b6e-predictor-7f59bbd875 namespace: kserve-ci-e2e-test resourceVersion: "21050" uid: 62c59c26-4dbf-4272-b202-e1359cead7bc kind: Event lastTimestamp: "2026-05-20T15:55:39Z" message: 'Created pod: error-404-isvc-80b6e-predictor-7f59bbd875-8td9x' metadata: creationTimestamp: "2026-05-20T15:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:55:39Z" name: error-404-isvc-80b6e-predictor-7f59bbd875.18b1512350e6671f namespace: kserve-ci-e2e-test resourceVersion: "21058" uid: cd4dd35f-109d-43c5-b52d-3969670fef47 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-80b6e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21048" uid: 857b3a27-9c86-467c-a3ce-4e33147c8c8f kind: Event lastTimestamp: "2026-05-20T15:55:39Z" message: Scaled up replica set error-404-isvc-80b6e-predictor-7f59bbd875 from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:55:39Z" name: error-404-isvc-80b6e-predictor.18b151234ec45601 namespace: kserve-ci-e2e-test resourceVersion: "21052" uid: 30657afe-b5e6-4b60-8781-4c91d9bfc741 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:55:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-80b6e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21049" uid: 527d9ac9-3831-477e-b08a-dd4e05447d19 kind: Event lastTimestamp: "2026-05-20T15:56:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:55:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:56:24Z" name: error-404-isvc-80b6e-predictor.18b15126cd85a670 namespace: kserve-ci-e2e-test resourceVersion: "21489" uid: 71bd0ee9-a4c1-468d-a5d0-c84e6eeab67b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:55:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-80b6e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21049" uid: 527d9ac9-3831-477e-b08a-dd4e05447d19 kind: Event lastTimestamp: "2026-05-20T15:56:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:55:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:56:24Z" name: error-404-isvc-80b6e-predictor.18b15126cdda1fe6 namespace: kserve-ci-e2e-test resourceVersion: "21491" uid: 3ab2fb17-84c6-4c73-8975-764e57aea658 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:56:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-80b6e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21236" uid: 527d9ac9-3831-477e-b08a-dd4e05447d19 kind: Event lastTimestamp: "2026-05-20T15:57:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:56:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:57:09Z" name: error-404-isvc-80b6e-predictor.18b151314b29a7a0 namespace: kserve-ci-e2e-test resourceVersion: "21929" uid: ab11523d-fd41-4560-9639-1072ea5c4777 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:56:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-80b6e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21236" uid: 527d9ac9-3831-477e-b08a-dd4e05447d19 kind: Event lastTimestamp: "2026-05-20T15:57:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:56:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:57:09Z" name: error-404-isvc-80b6e-predictor.18b151314bbc45a9 namespace: kserve-ci-e2e-test resourceVersion: "21930" uid: ee1711f2-14b0-4858-b5c4-6787a60261fe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T15:55:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-80b6e namespace: kserve-ci-e2e-test resourceVersion: "21027" uid: ba1908b5-8fca-487b-8574-84fe4314012b kind: Event lastTimestamp: "2026-05-20T15:56:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:56:37Z" name: error-404-isvc-80b6e.18b151234e9b162a namespace: kserve-ci-e2e-test resourceVersion: "21636" uid: 222601b9-dc62-440d-a9bd-71bef5d8d53c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-80b6e namespace: kserve-ci-e2e-test resourceVersion: "21027" uid: ba1908b5-8fca-487b-8574-84fe4314012b kind: Event lastTimestamp: "2026-05-20T15:55:39Z" message: 'Failed to update status for InferenceService "error-404-isvc-80b6e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-80b6e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:55:39Z" name: error-404-isvc-80b6e.18b151234f129689 namespace: kserve-ci-e2e-test resourceVersion: "21054" uid: b9d8c566-5995-47eb-99ea-e6d82cffb5b7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-80b6e namespace: kserve-ci-e2e-test resourceVersion: "21027" uid: ba1908b5-8fca-487b-8574-84fe4314012b kind: Event lastTimestamp: "2026-05-20T15:55:39Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-80b6e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:55:39Z" name: error-404-isvc-80b6e.18b151234f12e5dd namespace: kserve-ci-e2e-test resourceVersion: "21055" uid: 61cd57c6-6668-4c0f-8fa7-ada614b0bbdb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:56:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-80b6e namespace: kserve-ci-e2e-test resourceVersion: "21631" uid: ba1908b5-8fca-487b-8574-84fe4314012b kind: Event lastTimestamp: "2026-05-20T15:56:37Z" message: InferenceService [error-404-isvc-80b6e] is Ready metadata: creationTimestamp: "2026-05-20T15:56:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:56:37Z" name: error-404-isvc-80b6e.18b15130d23b2da4 namespace: kserve-ci-e2e-test resourceVersion: "21632" uid: dfd8142e-aef7-40a2-aa54-1336cd11d942 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:44:39.105278Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d4d66-predictor-867b8b47f7-js4z5 namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: 4aa54823-53e0-4d1e-aafe-9c7f98cc3c0b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d4d66-predictor-867b8b47f7-js4z5 to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T15:44:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:44:39Z" name: error-404-isvc-d4d66-predictor-867b8b47f7-js4z5.18b1508975cee460 namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: bc322624-7907-4ee9-952b-ec728510b553 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:40Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d4d66-predictor-867b8b47f7-js4z5 namespace: kserve-ci-e2e-test resourceVersion: "15300" uid: 4aa54823-53e0-4d1e-aafe-9c7f98cc3c0b kind: Event lastTimestamp: "2026-05-20T15:44:40Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:44:40Z" name: error-404-isvc-d4d66-predictor-867b8b47f7-js4z5.18b15089c4b00195 namespace: kserve-ci-e2e-test resourceVersion: "15336" uid: 0207d5d3-d38b-44e9-bb9f-8972d82b01ba reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d4d66-predictor-867b8b47f7-js4z5 namespace: kserve-ci-e2e-test resourceVersion: "15297" uid: 4aa54823-53e0-4d1e-aafe-9c7f98cc3c0b kind: Event lastTimestamp: "2026-05-20T15:44:40Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-05-20T15:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:40Z" name: error-404-isvc-d4d66-predictor-867b8b47f7-js4z5.18b15089c6e3a728 namespace: kserve-ci-e2e-test resourceVersion: "15338" uid: 9e1a8f2a-4bff-437b-b428-c705d02fc3b8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d4d66-predictor-867b8b47f7-js4z5 namespace: kserve-ci-e2e-test resourceVersion: "15297" uid: 4aa54823-53e0-4d1e-aafe-9c7f98cc3c0b kind: Event lastTimestamp: "2026-05-20T15:44:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:40Z" name: error-404-isvc-d4d66-predictor-867b8b47f7-js4z5.18b15089c9efcb37 namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: 87a8c761-2811-4df5-bf4f-35510c49f510 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d4d66-predictor-867b8b47f7-js4z5 namespace: kserve-ci-e2e-test resourceVersion: "15297" uid: 4aa54823-53e0-4d1e-aafe-9c7f98cc3c0b kind: Event lastTimestamp: "2026-05-20T15:44:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:40Z" name: error-404-isvc-d4d66-predictor-867b8b47f7-js4z5.18b15089ca600510 namespace: kserve-ci-e2e-test resourceVersion: "15341" uid: 60c137ea-d519-4035-ad96-423db2d406f3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d4d66-predictor-867b8b47f7-js4z5 namespace: kserve-ci-e2e-test resourceVersion: "15297" uid: 4aa54823-53e0-4d1e-aafe-9c7f98cc3c0b kind: Event lastTimestamp: "2026-05-20T15:44:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T15:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:40Z" name: error-404-isvc-d4d66-predictor-867b8b47f7-js4z5.18b15089ca6c261e namespace: kserve-ci-e2e-test resourceVersion: "15342" uid: c6dbfeb1-1fa6-4cc3-b1ee-4e7e629d854f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d4d66-predictor-867b8b47f7-js4z5 namespace: kserve-ci-e2e-test resourceVersion: "15297" uid: 4aa54823-53e0-4d1e-aafe-9c7f98cc3c0b kind: Event lastTimestamp: "2026-05-20T15:44:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:40Z" name: error-404-isvc-d4d66-predictor-867b8b47f7-js4z5.18b15089ce26f312 namespace: kserve-ci-e2e-test resourceVersion: "15343" uid: 8a6213fa-ff35-4231-88a9-280a33f576b0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d4d66-predictor-867b8b47f7-js4z5 namespace: kserve-ci-e2e-test resourceVersion: "15297" uid: 4aa54823-53e0-4d1e-aafe-9c7f98cc3c0b kind: Event lastTimestamp: "2026-05-20T15:44:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:40Z" name: error-404-isvc-d4d66-predictor-867b8b47f7-js4z5.18b15089ce8b8e86 namespace: kserve-ci-e2e-test resourceVersion: "15344" uid: 3af06ba5-5c2a-4f69-9373-79a2bdc7555a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-20T15:44:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d4d66-predictor-867b8b47f7-js4z5 namespace: kserve-ci-e2e-test resourceVersion: "15297" uid: 4aa54823-53e0-4d1e-aafe-9c7f98cc3c0b kind: Event lastTimestamp: "2026-05-20T15:45:28Z" message: 'Readiness probe failed: dial tcp 10.132.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:44:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:28Z" name: error-404-isvc-d4d66-predictor-867b8b47f7-js4z5.18b1508a3643bc54 namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: 6ab6b4b6-43d5-4e0c-aa27-33fb235bd57c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d4d66-predictor-867b8b47f7-js4z5 namespace: kserve-ci-e2e-test resourceVersion: "15297" uid: 4aa54823-53e0-4d1e-aafe-9c7f98cc3c0b kind: Event lastTimestamp: "2026-05-20T15:54:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T15:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:03Z" name: error-404-isvc-d4d66-predictor-867b8b47f7-js4z5.18b1510ced53ceb7 namespace: kserve-ci-e2e-test resourceVersion: "19716" uid: f689c7dc-4a65-481a-8b58-0dccc11c88c8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d4d66-predictor-867b8b47f7-js4z5 namespace: kserve-ci-e2e-test resourceVersion: "15297" uid: 4aa54823-53e0-4d1e-aafe-9c7f98cc3c0b kind: Event lastTimestamp: "2026-05-20T15:54:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:03Z" name: error-404-isvc-d4d66-predictor-867b8b47f7-js4z5.18b1510ced54dddf namespace: kserve-ci-e2e-test resourceVersion: "19719" uid: c5559261-f2ce-49d3-9955-40f31421325a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d4d66-predictor-867b8b47f7 namespace: kserve-ci-e2e-test resourceVersion: "15289" uid: b7411564-4a7a-49b2-b79f-36674e9ac276 kind: Event lastTimestamp: "2026-05-20T15:44:39Z" message: 'Created pod: error-404-isvc-d4d66-predictor-867b8b47f7-js4z5' metadata: creationTimestamp: "2026-05-20T15:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:39Z" name: error-404-isvc-d4d66-predictor-867b8b47f7.18b15089738e98b2 namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 8c6cc857-5887-4ce5-a89b-2bc218abeee4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d4d66-predictor namespace: kserve-ci-e2e-test resourceVersion: "15287" uid: 7a1d55c5-7036-4fbd-804d-aada4ad9a7e7 kind: Event lastTimestamp: "2026-05-20T15:44:39Z" message: Scaled up replica set error-404-isvc-d4d66-predictor-867b8b47f7 from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:39Z" name: error-404-isvc-d4d66-predictor.18b150896fa4ac42 namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: c73db1c0-001a-498a-90ec-c300251ee07d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:44:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d4d66-predictor namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 4f5a2bc0-9c8a-45b9-ba8e-6f1169610064 kind: Event lastTimestamp: "2026-05-20T15:45:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:44:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:54Z" name: error-404-isvc-d4d66-predictor.18b1508cedfbb0ef namespace: kserve-ci-e2e-test resourceVersion: "16227" uid: a594ee65-098e-4cea-9af6-0f8ea05d3f8b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:44:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d4d66-predictor namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 4f5a2bc0-9c8a-45b9-ba8e-6f1169610064 kind: Event lastTimestamp: "2026-05-20T15:45:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:44:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:54Z" name: error-404-isvc-d4d66-predictor.18b1508cee85086d namespace: kserve-ci-e2e-test resourceVersion: "16228" uid: debed4dc-4931-4015-9269-8100cd309312 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d4d66-predictor namespace: kserve-ci-e2e-test resourceVersion: "15482" uid: 4f5a2bc0-9c8a-45b9-ba8e-6f1169610064 kind: Event lastTimestamp: "2026-05-20T15:45:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:45:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:09Z" name: error-404-isvc-d4d66-predictor.18b150906d8f3193 namespace: kserve-ci-e2e-test resourceVersion: "15636" uid: dc22f868-b28b-4302-a495-184ba88164ff reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d4d66-predictor namespace: kserve-ci-e2e-test resourceVersion: "15482" uid: 4f5a2bc0-9c8a-45b9-ba8e-6f1169610064 kind: Event lastTimestamp: "2026-05-20T15:45:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:45:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:09Z" name: error-404-isvc-d4d66-predictor.18b150906e0f03da namespace: kserve-ci-e2e-test resourceVersion: "15638" uid: 1c814f89-0511-4e20-b358-41b4f3c692fe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T15:44:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d4d66 namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: 2663b6e2-0aae-4550-8e95-208c0be12c8b kind: Event lastTimestamp: "2026-05-20T15:45:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:45:38Z" name: error-404-isvc-d4d66.18b150896f210e97 namespace: kserve-ci-e2e-test resourceVersion: "16064" uid: ed503cef-71d6-42ee-809b-7b3a1fd9d1dd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d4d66 namespace: kserve-ci-e2e-test resourceVersion: "16061" uid: 2663b6e2-0aae-4550-8e95-208c0be12c8b kind: Event lastTimestamp: "2026-05-20T15:45:38Z" message: InferenceService [error-404-isvc-d4d66] is Ready metadata: creationTimestamp: "2026-05-20T15:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:45:38Z" name: error-404-isvc-d4d66.18b150974663ddfe namespace: kserve-ci-e2e-test resourceVersion: "16062" uid: 9d4cc9c8-0314-4986-9bf3-a1eaa3dda1f3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T16:04:49.033342Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj namespace: kserve-ci-e2e-test resourceVersion: "25412" uid: 0eda94f7-0a45-42af-ace9-a99be2b1368c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T16:04:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T16:04:49Z" name: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj.18b151a32b23f0a3 namespace: kserve-ci-e2e-test resourceVersion: "25416" uid: 2b952b77-6ce4-410b-9268-bf74e1a07545 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:49Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj namespace: kserve-ci-e2e-test resourceVersion: "25413" uid: 0eda94f7-0a45-42af-ace9-a99be2b1368c kind: Event lastTimestamp: "2026-05-20T16:04:49Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-e05eb-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T16:04:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:04:49Z" name: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj.18b151a334f97eab namespace: kserve-ci-e2e-test resourceVersion: "25426" uid: e2eae90a-d633-4385-9042-43e1ffcd9c09 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:50Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj namespace: kserve-ci-e2e-test resourceVersion: "25421" uid: 0eda94f7-0a45-42af-ace9-a99be2b1368c kind: Event lastTimestamp: "2026-05-20T16:04:50Z" message: Add eth0 [10.132.0.58/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T16:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T16:04:50Z" name: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj.18b151a3699e3ae1 namespace: kserve-ci-e2e-test resourceVersion: "25444" uid: 8ca368c4-3519-491c-b6f0-976fa090a883 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj namespace: kserve-ci-e2e-test resourceVersion: "25413" uid: 0eda94f7-0a45-42af-ace9-a99be2b1368c kind: Event lastTimestamp: "2026-05-20T16:04:50Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-05-20T16:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:04:50Z" name: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj.18b151a36b105fc9 namespace: kserve-ci-e2e-test resourceVersion: "25451" uid: e1ffb0ad-b8ac-4906-b73b-1b65c5e7b916 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj namespace: kserve-ci-e2e-test resourceVersion: "25413" uid: 0eda94f7-0a45-42af-ace9-a99be2b1368c kind: Event lastTimestamp: "2026-05-20T16:04:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T16:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:04:50Z" name: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj.18b151a36def7da2 namespace: kserve-ci-e2e-test resourceVersion: "25453" uid: fc37e22a-8fd8-4ded-a6f1-afc57cac294f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj namespace: kserve-ci-e2e-test resourceVersion: "25413" uid: 0eda94f7-0a45-42af-ace9-a99be2b1368c kind: Event lastTimestamp: "2026-05-20T16:04:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T16:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:04:50Z" name: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj.18b151a36e57cf88 namespace: kserve-ci-e2e-test resourceVersion: "25454" uid: 25a23ed7-8529-44a8-9368-8f690e9cddd2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj namespace: kserve-ci-e2e-test resourceVersion: "25413" uid: 0eda94f7-0a45-42af-ace9-a99be2b1368c kind: Event lastTimestamp: "2026-05-20T16:04:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T16:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:04:50Z" name: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj.18b151a36e636ee5 namespace: kserve-ci-e2e-test resourceVersion: "25455" uid: 48515977-6200-439e-86e1-51e056e8a704 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj namespace: kserve-ci-e2e-test resourceVersion: "25413" uid: 0eda94f7-0a45-42af-ace9-a99be2b1368c kind: Event lastTimestamp: "2026-05-20T16:04:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T16:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:04:50Z" name: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj.18b151a371ebdbaa namespace: kserve-ci-e2e-test resourceVersion: "25458" uid: 53533451-d5e1-4e42-af03-68ae821de7ea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj namespace: kserve-ci-e2e-test resourceVersion: "25413" uid: 0eda94f7-0a45-42af-ace9-a99be2b1368c kind: Event lastTimestamp: "2026-05-20T16:04:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T16:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:04:50Z" name: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj.18b151a3725678d2 namespace: kserve-ci-e2e-test resourceVersion: "25459" uid: a7d0694d-d669-4a8c-a1c7-8a8b53a44980 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T16:04:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj namespace: kserve-ci-e2e-test resourceVersion: "25413" uid: 0eda94f7-0a45-42af-ace9-a99be2b1368c kind: Event lastTimestamp: "2026-05-20T16:05:27Z" message: 'Readiness probe failed: dial tcp 10.132.0.58:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T16:04:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:27Z" name: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj.18b151a3c9d63fd1 namespace: kserve-ci-e2e-test resourceVersion: "25934" uid: 33c79b14-d865-43a0-a01b-c316c9f1dbef reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj namespace: kserve-ci-e2e-test resourceVersion: "25413" uid: 0eda94f7-0a45-42af-ace9-a99be2b1368c kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:09Z" name: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj.18b151b5ddf4184d namespace: kserve-ci-e2e-test resourceVersion: "26417" uid: f7befd42-2b28-44ed-b27e-486e125f0ae1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj namespace: kserve-ci-e2e-test resourceVersion: "25413" uid: 0eda94f7-0a45-42af-ace9-a99be2b1368c kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:09Z" name: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj.18b151b5ddf51fa4 namespace: kserve-ci-e2e-test resourceVersion: "26419" uid: 091f91cb-d296-4384-b042-daba3522a420 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj namespace: kserve-ci-e2e-test resourceVersion: "25413" uid: 0eda94f7-0a45-42af-ace9-a99be2b1368c kind: Event lastTimestamp: "2026-05-20T16:06:12Z" message: 'Readiness probe failed: Get "https://10.132.0.58:8643/healthz": dial tcp 10.132.0.58:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T16:06:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:12Z" name: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj.18b151b6a62bb0dd namespace: kserve-ci-e2e-test resourceVersion: "26506" uid: 10fe8e19-2610-437b-88f6-c8aaee83ae87 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e05eb-predictor-7cf65457c5 namespace: kserve-ci-e2e-test resourceVersion: "25403" uid: 8fdf3fc6-6ff2-4aa1-a96d-3106cdde4ccd kind: Event lastTimestamp: "2026-05-20T16:04:49Z" message: 'Created pod: error-404-isvc-e05eb-predictor-7cf65457c5-6cgfj' metadata: creationTimestamp: "2026-05-20T16:04:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:04:49Z" name: error-404-isvc-e05eb-predictor-7cf65457c5.18b151a32ac95e71 namespace: kserve-ci-e2e-test resourceVersion: "25414" uid: e397b6c5-b3ec-4933-89c5-b2c5a5c55ae9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e05eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25399" uid: 088d3000-ccc2-48a0-95fb-a03458307f42 kind: Event lastTimestamp: "2026-05-20T16:04:48Z" message: Scaled up replica set error-404-isvc-e05eb-predictor-7cf65457c5 from 0 to 1 metadata: creationTimestamp: "2026-05-20T16:04:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:04:48Z" name: error-404-isvc-e05eb-predictor.18b151a328f8f176 namespace: kserve-ci-e2e-test resourceVersion: "25406" uid: b0f4f359-e04f-43a0-9b76-83f380273a68 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T16:05:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e05eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25402" uid: 3053f48b-5f94-4465-bbb5-61a99900eecc kind: Event lastTimestamp: "2026-05-20T16:05:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T16:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:05:34Z" name: error-404-isvc-e05eb-predictor.18b151a6a7b736d8 namespace: kserve-ci-e2e-test resourceVersion: "25982" uid: 8e7569e5-f0f0-4d38-b3f2-6d8c45d784cc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T16:05:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e05eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25402" uid: 3053f48b-5f94-4465-bbb5-61a99900eecc kind: Event lastTimestamp: "2026-05-20T16:05:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T16:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:05:34Z" name: error-404-isvc-e05eb-predictor.18b151a6a829e9a1 namespace: kserve-ci-e2e-test resourceVersion: "25983" uid: cebfaff4-b423-491d-ba5c-b61f2c6c7c99 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T16:05:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e05eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25578" uid: 3053f48b-5f94-4465-bbb5-61a99900eecc kind: Event lastTimestamp: "2026-05-20T16:06:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T16:05:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:06:04Z" name: error-404-isvc-e05eb-predictor.18b151b124ea4802 namespace: kserve-ci-e2e-test resourceVersion: "26292" uid: 803ef657-0feb-4c6d-82a0-d9440398ff6c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T16:05:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e05eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25578" uid: 3053f48b-5f94-4465-bbb5-61a99900eecc kind: Event lastTimestamp: "2026-05-20T16:06:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T16:05:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:06:04Z" name: error-404-isvc-e05eb-predictor.18b151b125dda393 namespace: kserve-ci-e2e-test resourceVersion: "26293" uid: 615cc518-6c61-408d-8a25-23a32da160f1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T16:04:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e05eb namespace: kserve-ci-e2e-test resourceVersion: "25373" uid: 43729553-db24-48eb-8163-573ac6ec7ab4 kind: Event lastTimestamp: "2026-05-20T16:05:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T16:04:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:05:37Z" name: error-404-isvc-e05eb.18b151a328deb585 namespace: kserve-ci-e2e-test resourceVersion: "26025" uid: 548251e0-b12f-4a56-b41f-7d7ee021b728 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e05eb namespace: kserve-ci-e2e-test resourceVersion: "25373" uid: 43729553-db24-48eb-8163-573ac6ec7ab4 kind: Event lastTimestamp: "2026-05-20T16:04:49Z" message: 'Failed to update status for InferenceService "error-404-isvc-e05eb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e05eb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T16:04:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:04:49Z" name: error-404-isvc-e05eb.18b151a32946d6a8 namespace: kserve-ci-e2e-test resourceVersion: "25409" uid: b6e9b9c7-f141-4978-afc6-5bf8f5ce9609 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e05eb namespace: kserve-ci-e2e-test resourceVersion: "25373" uid: 43729553-db24-48eb-8163-573ac6ec7ab4 kind: Event lastTimestamp: "2026-05-20T16:04:49Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e05eb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T16:04:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:04:49Z" name: error-404-isvc-e05eb.18b151a329473833 namespace: kserve-ci-e2e-test resourceVersion: "25410" uid: 974ed2f0-f0f3-4d0f-8270-592b85b8dd6f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e05eb namespace: kserve-ci-e2e-test resourceVersion: "26023" uid: 43729553-db24-48eb-8163-573ac6ec7ab4 kind: Event lastTimestamp: "2026-05-20T16:05:37Z" message: InferenceService [error-404-isvc-e05eb] is Ready metadata: creationTimestamp: "2026-05-20T16:05:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:05:37Z" name: error-404-isvc-e05eb.18b151ae85eea7f0 namespace: kserve-ci-e2e-test resourceVersion: "26024" uid: 750ff675-bdd4-496c-a27c-be097c1c8295 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T16:05:04.833427Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fd7d8-predictor-6dfb6b7f9-72m24 namespace: kserve-ci-e2e-test resourceVersion: "25679" uid: 84084650-0c60-412c-af89-602e1b8a8101 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-fd7d8-predictor-6dfb6b7f9-72m24 to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T16:05:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T16:05:04Z" name: error-404-isvc-fd7d8-predictor-6dfb6b7f9-72m24.18b151a6d8e620a4 namespace: kserve-ci-e2e-test resourceVersion: "25683" uid: b88235c5-cf8a-4ab6-a118-f5310574322b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:05Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fd7d8-predictor-6dfb6b7f9-72m24 namespace: kserve-ci-e2e-test resourceVersion: "25686" uid: 84084650-0c60-412c-af89-602e1b8a8101 kind: Event lastTimestamp: "2026-05-20T16:05:05Z" message: Add eth0 [10.132.0.60/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T16:05:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T16:05:05Z" name: error-404-isvc-fd7d8-predictor-6dfb6b7f9-72m24.18b151a6f3426d72 namespace: kserve-ci-e2e-test resourceVersion: "25701" uid: feda340a-b93f-4e65-9f6b-204869f39e4f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fd7d8-predictor-6dfb6b7f9-72m24 namespace: kserve-ci-e2e-test resourceVersion: "25681" uid: 84084650-0c60-412c-af89-602e1b8a8101 kind: Event lastTimestamp: "2026-05-20T16:05:05Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-05-20T16:05:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:05Z" name: error-404-isvc-fd7d8-predictor-6dfb6b7f9-72m24.18b151a6f4a9ec95 namespace: kserve-ci-e2e-test resourceVersion: "25704" uid: 9a790d11-ca5f-49a2-87e7-ef98b769578e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fd7d8-predictor-6dfb6b7f9-72m24 namespace: kserve-ci-e2e-test resourceVersion: "25681" uid: 84084650-0c60-412c-af89-602e1b8a8101 kind: Event lastTimestamp: "2026-05-20T16:05:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T16:05:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:05Z" name: error-404-isvc-fd7d8-predictor-6dfb6b7f9-72m24.18b151a6f7fe4dce namespace: kserve-ci-e2e-test resourceVersion: "25705" uid: d82a8e04-8f4a-49b5-98e3-5eeffde48592 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fd7d8-predictor-6dfb6b7f9-72m24 namespace: kserve-ci-e2e-test resourceVersion: "25681" uid: 84084650-0c60-412c-af89-602e1b8a8101 kind: Event lastTimestamp: "2026-05-20T16:05:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T16:05:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:05Z" name: error-404-isvc-fd7d8-predictor-6dfb6b7f9-72m24.18b151a6f86ae16c namespace: kserve-ci-e2e-test resourceVersion: "25706" uid: 61341e3f-7eca-470a-b3a3-b1c5ad7d5820 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fd7d8-predictor-6dfb6b7f9-72m24 namespace: kserve-ci-e2e-test resourceVersion: "25681" uid: 84084650-0c60-412c-af89-602e1b8a8101 kind: Event lastTimestamp: "2026-05-20T16:05:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T16:05:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:05Z" name: error-404-isvc-fd7d8-predictor-6dfb6b7f9-72m24.18b151a6f8779ef2 namespace: kserve-ci-e2e-test resourceVersion: "25707" uid: 71f48bd3-0c72-4be2-bdaa-e7d3854e361b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fd7d8-predictor-6dfb6b7f9-72m24 namespace: kserve-ci-e2e-test resourceVersion: "25681" uid: 84084650-0c60-412c-af89-602e1b8a8101 kind: Event lastTimestamp: "2026-05-20T16:05:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T16:05:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:05Z" name: error-404-isvc-fd7d8-predictor-6dfb6b7f9-72m24.18b151a6fc11a179 namespace: kserve-ci-e2e-test resourceVersion: "25709" uid: a02705ba-567d-4e8a-8bc6-31a5cb405f0c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fd7d8-predictor-6dfb6b7f9-72m24 namespace: kserve-ci-e2e-test resourceVersion: "25681" uid: 84084650-0c60-412c-af89-602e1b8a8101 kind: Event lastTimestamp: "2026-05-20T16:05:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T16:05:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:05Z" name: error-404-isvc-fd7d8-predictor-6dfb6b7f9-72m24.18b151a6fc801a8c namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: 9af61a53-0e07-4e6a-9bb5-faf411600f59 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-20T16:05:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fd7d8-predictor-6dfb6b7f9-72m24 namespace: kserve-ci-e2e-test resourceVersion: "25681" uid: 84084650-0c60-412c-af89-602e1b8a8101 kind: Event lastTimestamp: "2026-05-20T16:05:52Z" message: 'Readiness probe failed: dial tcp 10.132.0.60:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T16:05:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:52Z" name: error-404-isvc-fd7d8-predictor-6dfb6b7f9-72m24.18b151a74be7d41d namespace: kserve-ci-e2e-test resourceVersion: "26152" uid: 04b5566b-de83-4bae-83ef-bd329825698a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:22:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fd7d8-predictor-6dfb6b7f9-72m24 namespace: kserve-ci-e2e-test resourceVersion: "25681" uid: 84084650-0c60-412c-af89-602e1b8a8101 kind: Event lastTimestamp: "2026-05-20T16:22:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T16:22:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:22:34Z" name: error-404-isvc-fd7d8-predictor-6dfb6b7f9-72m24.18b1529b317ef65e namespace: kserve-ci-e2e-test resourceVersion: "33495" uid: 4f5f0113-c829-4b6b-acb9-d263bc368ae6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:22:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fd7d8-predictor-6dfb6b7f9-72m24 namespace: kserve-ci-e2e-test resourceVersion: "25681" uid: 84084650-0c60-412c-af89-602e1b8a8101 kind: Event lastTimestamp: "2026-05-20T16:22:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T16:22:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:22:34Z" name: error-404-isvc-fd7d8-predictor-6dfb6b7f9-72m24.18b1529b317f9842 namespace: kserve-ci-e2e-test resourceVersion: "33496" uid: f8362aac-9157-4b8f-a7ac-a0010d653d99 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-fd7d8-predictor-6dfb6b7f9 namespace: kserve-ci-e2e-test resourceVersion: "25672" uid: 471e9a51-3788-493b-96ae-fb0e2758a7a7 kind: Event lastTimestamp: "2026-05-20T16:05:04Z" message: 'Created pod: error-404-isvc-fd7d8-predictor-6dfb6b7f9-72m24' metadata: creationTimestamp: "2026-05-20T16:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:05:04Z" name: error-404-isvc-fd7d8-predictor-6dfb6b7f9.18b151a6d881cc38 namespace: kserve-ci-e2e-test resourceVersion: "25680" uid: 853190a5-6047-48c1-9dae-e962f4a6d851 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-fd7d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "25670" uid: c23f52d6-6e4e-400a-8469-48809533dc5f kind: Event lastTimestamp: "2026-05-20T16:05:04Z" message: Scaled up replica set error-404-isvc-fd7d8-predictor-6dfb6b7f9 from 0 to 1 metadata: creationTimestamp: "2026-05-20T16:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:05:04Z" name: error-404-isvc-fd7d8-predictor.18b151a6d6c727a2 namespace: kserve-ci-e2e-test resourceVersion: "25673" uid: 8fe81ce1-3718-4d70-b0eb-7b801c7ab12c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T16:05:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fd7d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "25671" uid: e0e66fac-0ca4-4efb-a1d9-000356b385de kind: Event lastTimestamp: "2026-05-20T16:05:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T16:05:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:05:34Z" name: error-404-isvc-fd7d8-predictor.18b151aa55af83b5 namespace: kserve-ci-e2e-test resourceVersion: "25993" uid: c2dc53db-e46e-4b4e-b161-89b875b58fff reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T16:05:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fd7d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "25671" uid: e0e66fac-0ca4-4efb-a1d9-000356b385de kind: Event lastTimestamp: "2026-05-20T16:05:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T16:05:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:05:34Z" name: error-404-isvc-fd7d8-predictor.18b151aa56109e6d namespace: kserve-ci-e2e-test resourceVersion: "25994" uid: 59f0e4ee-99cf-4483-9d69-477ed0a8a87d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T16:05:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fd7d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "25870" uid: e0e66fac-0ca4-4efb-a1d9-000356b385de kind: Event lastTimestamp: "2026-05-20T16:06:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T16:05:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:06:34Z" name: error-404-isvc-fd7d8-predictor.18b151b1542b0729 namespace: kserve-ci-e2e-test resourceVersion: "26731" uid: 125c1ccd-ce1f-414f-9917-1b04ec448b14 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T16:05:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fd7d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "25870" uid: e0e66fac-0ca4-4efb-a1d9-000356b385de kind: Event lastTimestamp: "2026-05-20T16:06:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T16:05:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:06:34Z" name: error-404-isvc-fd7d8-predictor.18b151b154a41dce namespace: kserve-ci-e2e-test resourceVersion: "26732" uid: 2ccabfed-4a3a-452d-a4f5-d810793ca8f9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T16:05:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fd7d8 namespace: kserve-ci-e2e-test resourceVersion: "25654" uid: 129071ab-f1b8-4064-a663-09015f312c6b kind: Event lastTimestamp: "2026-05-20T16:06:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T16:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:06:02Z" name: error-404-isvc-fd7d8.18b151a6d6d18baf namespace: kserve-ci-e2e-test resourceVersion: "26273" uid: 76363a24-c6a1-4962-bc05-9c32a383e773 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fd7d8 namespace: kserve-ci-e2e-test resourceVersion: "25654" uid: 129071ab-f1b8-4064-a663-09015f312c6b kind: Event lastTimestamp: "2026-05-20T16:05:04Z" message: 'Failed to update status for InferenceService "error-404-isvc-fd7d8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fd7d8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T16:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:05:04Z" name: error-404-isvc-fd7d8.18b151a6d768c5da namespace: kserve-ci-e2e-test resourceVersion: "25676" uid: 3f1c4825-0e15-46f7-96bd-01964b46e680 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fd7d8 namespace: kserve-ci-e2e-test resourceVersion: "25654" uid: 129071ab-f1b8-4064-a663-09015f312c6b kind: Event lastTimestamp: "2026-05-20T16:05:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fd7d8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T16:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:05:04Z" name: error-404-isvc-fd7d8.18b151a6d769135f namespace: kserve-ci-e2e-test resourceVersion: "25678" uid: 3585164c-cb89-409c-9970-a04635df0246 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fd7d8 namespace: kserve-ci-e2e-test resourceVersion: "26269" uid: 129071ab-f1b8-4064-a663-09015f312c6b kind: Event lastTimestamp: "2026-05-20T16:06:02Z" message: InferenceService [error-404-isvc-fd7d8] is Ready metadata: creationTimestamp: "2026-05-20T16:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:06:02Z" name: error-404-isvc-fd7d8.18b151b45fa8de42 namespace: kserve-ci-e2e-test resourceVersion: "26271" uid: d67722ec-1f40-4a0f-923e-e711abf68d1d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:43:04.515262Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc namespace: kserve-ci-e2e-test resourceVersion: "14156" uid: d3d837f9-3a7c-4671-b758-5bac2d96320d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-6c6997cc58-829wc to ip-10-0-135-174.ec2.internal metadata: creationTimestamp: "2026-05-20T15:43:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:43:04Z" name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc.18b150736fcdc4b2 namespace: kserve-ci-e2e-test resourceVersion: "14164" uid: d206ba62-2006-406d-83e1-22ef560e6fd9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:04Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc namespace: kserve-ci-e2e-test resourceVersion: "14159" uid: d3d837f9-3a7c-4671-b758-5bac2d96320d kind: Event lastTimestamp: "2026-05-20T15:43:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T15:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:04Z" name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc.18b150737e4cdbc8 namespace: kserve-ci-e2e-test resourceVersion: "14218" uid: 5ac94dad-e7c5-430b-ab55-ab3e28bf962e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc namespace: kserve-ci-e2e-test resourceVersion: "14174" uid: d3d837f9-3a7c-4671-b758-5bac2d96320d kind: Event lastTimestamp: "2026-05-20T15:43:05Z" message: Add eth0 [10.134.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:43:05Z" name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc.18b15073ad7e89fe namespace: kserve-ci-e2e-test resourceVersion: "14261" uid: f5e6988a-9b7d-4285-b349-34330e0438b5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc namespace: kserve-ci-e2e-test resourceVersion: "14159" uid: d3d837f9-3a7c-4671-b758-5bac2d96320d kind: Event lastTimestamp: "2026-05-20T15:43:05Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:050b18baea3a969ddc2a15671f898a5f40ff53eb89f54c18856b130dc10bd894" metadata: creationTimestamp: "2026-05-20T15:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:05Z" name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc.18b15073aefe7e82 namespace: kserve-ci-e2e-test resourceVersion: "14263" uid: 9d011bf0-ac01-4dd2-894e-5d6135c79c98 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc namespace: kserve-ci-e2e-test resourceVersion: "14159" uid: d3d837f9-3a7c-4671-b758-5bac2d96320d kind: Event lastTimestamp: "2026-05-20T15:43:08Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:050b18baea3a969ddc2a15671f898a5f40ff53eb89f54c18856b130dc10bd894" in 3.191s (3.191s including waiting). Image size: 301471312 bytes.' metadata: creationTimestamp: "2026-05-20T15:43:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:08Z" name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc.18b150746d395f0d namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: a48651fa-290e-4a0d-a811-68b5ac43c0c9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc namespace: kserve-ci-e2e-test resourceVersion: "14159" uid: d3d837f9-3a7c-4671-b758-5bac2d96320d kind: Event lastTimestamp: "2026-05-20T15:43:08Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T15:43:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:08Z" name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc.18b15074707045a2 namespace: kserve-ci-e2e-test resourceVersion: "14309" uid: 49dcf15f-5178-42fa-a968-8f0b05ac2fa7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc namespace: kserve-ci-e2e-test resourceVersion: "14159" uid: d3d837f9-3a7c-4671-b758-5bac2d96320d kind: Event lastTimestamp: "2026-05-20T15:43:08Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T15:43:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:08Z" name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc.18b1507470d29124 namespace: kserve-ci-e2e-test resourceVersion: "14311" uid: fc2585ae-2a15-4272-ab29-62a57261c085 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc namespace: kserve-ci-e2e-test resourceVersion: "14159" uid: d3d837f9-3a7c-4671-b758-5bac2d96320d kind: Event lastTimestamp: "2026-05-20T15:43:12Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1433" metadata: creationTimestamp: "2026-05-20T15:43:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:12Z" name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc.18b1507554ae0693 namespace: kserve-ci-e2e-test resourceVersion: "14360" uid: 32e3c084-72a9-49f6-9fb8-e78e9c252487 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc namespace: kserve-ci-e2e-test resourceVersion: "14159" uid: d3d837f9-3a7c-4671-b758-5bac2d96320d kind: Event lastTimestamp: "2026-05-20T15:43:26Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1433" in 14.256s (14.256s including waiting). Image size: 1560928674 bytes.' metadata: creationTimestamp: "2026-05-20T15:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:26Z" name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc.18b15078a66bcc18 namespace: kserve-ci-e2e-test resourceVersion: "14507" uid: fd339c9b-5319-4773-b852-910924961f15 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc namespace: kserve-ci-e2e-test resourceVersion: "14159" uid: d3d837f9-3a7c-4671-b758-5bac2d96320d kind: Event lastTimestamp: "2026-05-20T15:43:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:26Z" name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc.18b15078aa1af8ec namespace: kserve-ci-e2e-test resourceVersion: "14510" uid: 86fdb9ff-528c-4dc9-a5ea-f07679954f21 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc namespace: kserve-ci-e2e-test resourceVersion: "14159" uid: d3d837f9-3a7c-4671-b758-5bac2d96320d kind: Event lastTimestamp: "2026-05-20T15:43:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:26Z" name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc.18b15078aa96e01b namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: 143f338d-57d1-430c-9d98-3e7a984221f2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc namespace: kserve-ci-e2e-test resourceVersion: "14159" uid: d3d837f9-3a7c-4671-b758-5bac2d96320d kind: Event lastTimestamp: "2026-05-20T15:43:26Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-20T15:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:26Z" name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc.18b15078aaa3b566 namespace: kserve-ci-e2e-test resourceVersion: "14513" uid: 84d988f2-3223-4e38-a267-687392f4a965 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc namespace: kserve-ci-e2e-test resourceVersion: "14159" uid: d3d837f9-3a7c-4671-b758-5bac2d96320d kind: Event lastTimestamp: "2026-05-20T15:43:29Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.412s (2.412s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-20T15:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:29Z" name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc.18b150793a692e0a namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: ed2f24f5-1b32-41ac-9dc0-d07c03c05afc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc namespace: kserve-ci-e2e-test resourceVersion: "14159" uid: d3d837f9-3a7c-4671-b758-5bac2d96320d kind: Event lastTimestamp: "2026-05-20T15:43:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:29Z" name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc.18b150793deeff97 namespace: kserve-ci-e2e-test resourceVersion: "14538" uid: c7b975cb-7c64-4a2c-87e5-29c590cb8ff2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc namespace: kserve-ci-e2e-test resourceVersion: "14159" uid: d3d837f9-3a7c-4671-b758-5bac2d96320d kind: Event lastTimestamp: "2026-05-20T15:43:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:29Z" name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc.18b150793e768e8d namespace: kserve-ci-e2e-test resourceVersion: "14540" uid: 7f367e1e-383b-469d-be5e-86193491f937 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-20T15:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc namespace: kserve-ci-e2e-test resourceVersion: "14159" uid: d3d837f9-3a7c-4671-b758-5bac2d96320d kind: Event lastTimestamp: "2026-05-20T15:45:15Z" message: 'Readiness probe failed: dial tcp 10.134.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:15Z" name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc.18b150794e9610dd namespace: kserve-ci-e2e-test resourceVersion: "15842" uid: 8800798c-0be2-4a65-ba95-54859bbe8678 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc namespace: kserve-ci-e2e-test resourceVersion: "14159" uid: d3d837f9-3a7c-4671-b758-5bac2d96320d kind: Event lastTimestamp: "2026-05-20T15:45:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T15:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:14Z" name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc.18b15091b8f65841 namespace: kserve-ci-e2e-test resourceVersion: "15761" uid: 9a5eb292-3655-40c8-b442-9de9eed1d056 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc namespace: kserve-ci-e2e-test resourceVersion: "14159" uid: d3d837f9-3a7c-4671-b758-5bac2d96320d kind: Event lastTimestamp: "2026-05-20T15:45:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:14Z" name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc.18b15091b8f6c1bb namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: c4dccfa4-b4b1-4d2c-b773-a30261bb621a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc namespace: kserve-ci-e2e-test resourceVersion: "14159" uid: d3d837f9-3a7c-4671-b758-5bac2d96320d kind: Event lastTimestamp: "2026-05-20T15:45:15Z" message: 'Readiness probe failed: Get "https://10.134.0.19:8643/healthz": dial tcp 10.134.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:15Z" name: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc.18b15091fccc3841 namespace: kserve-ci-e2e-test resourceVersion: "15841" uid: e2a60b36-5d01-4b35-9a65-cc75ee8d15e5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-6c6997cc58 namespace: kserve-ci-e2e-test resourceVersion: "14145" uid: e6212689-b761-4a94-8a71-a1cd99f53997 kind: Event lastTimestamp: "2026-05-20T15:43:04Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-6c6997cc58-829wc' metadata: creationTimestamp: "2026-05-20T15:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:43:04Z" name: isvc-sklearn-graph-1-predictor-6c6997cc58.18b150736f27f109 namespace: kserve-ci-e2e-test resourceVersion: "14158" uid: a0e5fa03-dcee-46eb-b7b9-13614b613a9c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14143" uid: 5482d85a-421b-4f5d-9f90-524a4b388549 kind: Event lastTimestamp: "2026-05-20T15:43:04Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-6c6997cc58 from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:43:04Z" name: isvc-sklearn-graph-1-predictor.18b150736a7a4773 namespace: kserve-ci-e2e-test resourceVersion: "14147" uid: c0e5ba69-605b-4f88-9155-92c108dd4f8c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:43:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14144" uid: e581238b-d118-43d7-a2fc-d77f706552f0 kind: Event lastTimestamp: "2026-05-20T15:44:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:04Z" name: isvc-sklearn-graph-1-predictor.18b15076e90674a4 namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 66459cfb-ca6a-48d5-b7ef-a2274c161ee2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:43:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14144" uid: e581238b-d118-43d7-a2fc-d77f706552f0 kind: Event lastTimestamp: "2026-05-20T15:44:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:04Z" name: isvc-sklearn-graph-1-predictor.18b15076e9b444ed namespace: kserve-ci-e2e-test resourceVersion: "14845" uid: 5d3e5ed6-edf9-4c3d-9882-9b57ea4026bb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:44:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14424" uid: e581238b-d118-43d7-a2fc-d77f706552f0 kind: Event lastTimestamp: "2026-05-20T15:45:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:04Z" name: isvc-sklearn-graph-1-predictor.18b15084e5e4d9d4 namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: e15cc583-23f4-4217-86b7-20b47ff78ac2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:44:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14424" uid: e581238b-d118-43d7-a2fc-d77f706552f0 kind: Event lastTimestamp: "2026-05-20T15:45:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:04Z" name: isvc-sklearn-graph-1-predictor.18b15084e6885320 namespace: kserve-ci-e2e-test resourceVersion: "15588" uid: 36aa6fe8-6ec3-4041-a0a7-20ead9c7b275 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-20T15:43:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14129" uid: 01e2a938-6772-4358-ba14-108f1ea97d34 kind: Event lastTimestamp: "2026-05-20T15:44:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:44:35Z" name: isvc-sklearn-graph-1.18b150736a284928 namespace: kserve-ci-e2e-test resourceVersion: "15173" uid: feb59dd7-ab1c-4de4-99b6-92f2adfd4811 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15171" uid: 01e2a938-6772-4358-ba14-108f1ea97d34 kind: Event lastTimestamp: "2026-05-20T15:44:35Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-05-20T15:44:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:44:35Z" name: isvc-sklearn-graph-1.18b15088b380ee02 namespace: kserve-ci-e2e-test resourceVersion: "15172" uid: 9fc97e27-9b93-467a-b181-2c318f21569e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:43:05.521916Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs namespace: kserve-ci-e2e-test resourceVersion: "14252" uid: b827a94d-93ca-4260-a3e6-3b3a429f8789 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs to ip-10-0-135-174.ec2.internal metadata: creationTimestamp: "2026-05-20T15:43:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:43:05Z" name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs.18b15073abce007a namespace: kserve-ci-e2e-test resourceVersion: "14256" uid: 871850d1-e269-4192-95c5-940674d9f256 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: b827a94d-93ca-4260-a3e6-3b3a429f8789 kind: Event lastTimestamp: "2026-05-20T15:43:05Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:43:05Z" name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs.18b15073c6416759 namespace: kserve-ci-e2e-test resourceVersion: "14268" uid: 96ea9640-001b-4f67-a529-ba160f94b191 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: b827a94d-93ca-4260-a3e6-3b3a429f8789 kind: Event lastTimestamp: "2026-05-20T15:43:05Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:050b18baea3a969ddc2a15671f898a5f40ff53eb89f54c18856b130dc10bd894" metadata: creationTimestamp: "2026-05-20T15:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:05Z" name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs.18b15073c809aa05 namespace: kserve-ci-e2e-test resourceVersion: "14271" uid: bf3a3b0d-63e1-4afe-9199-c3d7a8c5b9e8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: b827a94d-93ca-4260-a3e6-3b3a429f8789 kind: Event lastTimestamp: "2026-05-20T15:43:08Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:050b18baea3a969ddc2a15671f898a5f40ff53eb89f54c18856b130dc10bd894" in 2.761s (2.761s including waiting). Image size: 301471312 bytes.' metadata: creationTimestamp: "2026-05-20T15:43:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:08Z" name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs.18b150746ca5f4f9 namespace: kserve-ci-e2e-test resourceVersion: "14305" uid: 0f318a33-0176-4be5-ba58-572b6a8cb75a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: b827a94d-93ca-4260-a3e6-3b3a429f8789 kind: Event lastTimestamp: "2026-05-20T15:43:08Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T15:43:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:08Z" name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs.18b15074705aa0d2 namespace: kserve-ci-e2e-test resourceVersion: "14308" uid: 76927045-8366-4c95-9b54-0cc198ee843f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: b827a94d-93ca-4260-a3e6-3b3a429f8789 kind: Event lastTimestamp: "2026-05-20T15:43:08Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T15:43:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:08Z" name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs.18b1507470c6679b namespace: kserve-ci-e2e-test resourceVersion: "14310" uid: d8db4cc2-c2f9-4a17-97a1-383d9776f80f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: b827a94d-93ca-4260-a3e6-3b3a429f8789 kind: Event lastTimestamp: "2026-05-20T15:43:12Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1433" metadata: creationTimestamp: "2026-05-20T15:43:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:12Z" name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs.18b1507554c0d285 namespace: kserve-ci-e2e-test resourceVersion: "14361" uid: cf6b26a3-f516-490e-b861-1b4d1dcb1c72 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: b827a94d-93ca-4260-a3e6-3b3a429f8789 kind: Event lastTimestamp: "2026-05-20T15:43:26Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1433" in 14.262s (14.262s including waiting). Image size: 1560928674 bytes.' metadata: creationTimestamp: "2026-05-20T15:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:26Z" name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs.18b15078a6d8a422 namespace: kserve-ci-e2e-test resourceVersion: "14508" uid: 9d329cfb-b051-4379-a42b-ec276b6038ff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: b827a94d-93ca-4260-a3e6-3b3a429f8789 kind: Event lastTimestamp: "2026-05-20T15:43:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:26Z" name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs.18b15078aa2610c3 namespace: kserve-ci-e2e-test resourceVersion: "14511" uid: 55163657-b055-42dd-812e-35c55d346da9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: b827a94d-93ca-4260-a3e6-3b3a429f8789 kind: Event lastTimestamp: "2026-05-20T15:43:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:27Z" name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs.18b15078aaa50e22 namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: e94fcd2d-cf74-4012-92a2-4467051d1504 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: b827a94d-93ca-4260-a3e6-3b3a429f8789 kind: Event lastTimestamp: "2026-05-20T15:43:26Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-20T15:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:27Z" name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs.18b15078aab1bd4e namespace: kserve-ci-e2e-test resourceVersion: "14515" uid: 0610a93c-12da-4038-8538-51236a342cc5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: b827a94d-93ca-4260-a3e6-3b3a429f8789 kind: Event lastTimestamp: "2026-05-20T15:43:29Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.414s (2.414s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-20T15:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:29Z" name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs.18b150793a992c58 namespace: kserve-ci-e2e-test resourceVersion: "14536" uid: f0676a3e-fb62-4dc3-91c8-b81490da6aee reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: b827a94d-93ca-4260-a3e6-3b3a429f8789 kind: Event lastTimestamp: "2026-05-20T15:43:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:29Z" name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs.18b150793e06948d namespace: kserve-ci-e2e-test resourceVersion: "14539" uid: baad8615-056c-456d-a93e-2f32643747c9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: b827a94d-93ca-4260-a3e6-3b3a429f8789 kind: Event lastTimestamp: "2026-05-20T15:43:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:43:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:29Z" name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs.18b150793e81c690 namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: 4b81365c-aa97-41b8-9f95-bdb8710f3f56 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-20T15:43:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: b827a94d-93ca-4260-a3e6-3b3a429f8789 kind: Event lastTimestamp: "2026-05-20T15:45:16Z" message: 'Readiness probe failed: dial tcp 10.134.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:16Z" name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs.18b150798a5831f2 namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: 11df1082-9051-4a09-868a-04d851122e6d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: b827a94d-93ca-4260-a3e6-3b3a429f8789 kind: Event lastTimestamp: "2026-05-20T15:45:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T15:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:14Z" name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs.18b15091b4c98efb namespace: kserve-ci-e2e-test resourceVersion: "15734" uid: 2875ba7a-bc1c-43c4-abd6-e016eb19dbbf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: b827a94d-93ca-4260-a3e6-3b3a429f8789 kind: Event lastTimestamp: "2026-05-20T15:45:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:14Z" name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs.18b15091b4ca12fe namespace: kserve-ci-e2e-test resourceVersion: "15739" uid: 38976796-68e2-4c38-a5b8-e81ab15f8a95 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: b827a94d-93ca-4260-a3e6-3b3a429f8789 kind: Event lastTimestamp: "2026-05-20T15:45:16Z" message: 'Readiness probe failed: Get "https://10.134.0.20:8643/healthz": dial tcp 10.134.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:16Z" name: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs.18b15092388dab16 namespace: kserve-ci-e2e-test resourceVersion: "15859" uid: 1e388102-f282-4b63-9f62-9c4fc7bfa4ac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-58b5c9bf58 namespace: kserve-ci-e2e-test resourceVersion: "14238" uid: 3850a914-d8ec-4a7f-a809-7b680f25fd48 kind: Event lastTimestamp: "2026-05-20T15:43:05Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-58b5c9bf58-tk8zs' metadata: creationTimestamp: "2026-05-20T15:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:43:05Z" name: isvc-sklearn-graph-2-predictor-58b5c9bf58.18b15073ab4c379e namespace: kserve-ci-e2e-test resourceVersion: "14254" uid: d2d2f8a0-f0c4-44b6-bbad-0579fbef1c6d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14236" uid: 0b03aed5-ed91-4171-aba9-e28f65092073 kind: Event lastTimestamp: "2026-05-20T15:43:05Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-58b5c9bf58 from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:43:05Z" name: isvc-sklearn-graph-2-predictor.18b15073934fafcb namespace: kserve-ci-e2e-test resourceVersion: "14242" uid: bada2e27-2db5-4d29-86d6-76066f8c4a8e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:43:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14237" uid: 27b606d8-d025-437a-84b9-3f7665929c9d kind: Event lastTimestamp: "2026-05-20T15:44:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:43:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:05Z" name: isvc-sklearn-graph-2-predictor.18b1507712185f1f namespace: kserve-ci-e2e-test resourceVersion: "14856" uid: c03ec5f1-ab97-4beb-8ac4-36d1e9c85b16 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:43:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14237" uid: 27b606d8-d025-437a-84b9-3f7665929c9d kind: Event lastTimestamp: "2026-05-20T15:44:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:43:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:05Z" name: isvc-sklearn-graph-2-predictor.18b1507712894b31 namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 8dc83ef3-c42b-4b37-a27f-e112ba290d00 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:44:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14434" uid: 27b606d8-d025-437a-84b9-3f7665929c9d kind: Event lastTimestamp: "2026-05-20T15:45:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:05Z" name: isvc-sklearn-graph-2-predictor.18b150850f77a2e3 namespace: kserve-ci-e2e-test resourceVersion: "15594" uid: d05c4213-574b-4700-98f3-78bb3b301a1a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:44:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14434" uid: 27b606d8-d025-437a-84b9-3f7665929c9d kind: Event lastTimestamp: "2026-05-20T15:45:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:05Z" name: isvc-sklearn-graph-2-predictor.18b150850fee1444 namespace: kserve-ci-e2e-test resourceVersion: "15596" uid: 09632a00-aaee-4be1-b9b4-8ad75ed9609f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-20T15:43:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14220" uid: f0c1712c-90dc-433c-bb36-0b0a7d86dc5e kind: Event lastTimestamp: "2026-05-20T15:44:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:44:36Z" name: isvc-sklearn-graph-2.18b150739318a6ab namespace: kserve-ci-e2e-test resourceVersion: "15186" uid: 8ae5aa43-83f3-4afc-a456-860e920c4f7e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15182" uid: f0c1712c-90dc-433c-bb36-0b0a7d86dc5e kind: Event lastTimestamp: "2026-05-20T15:44:36Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-05-20T15:44:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:44:36Z" name: isvc-sklearn-graph-2.18b15088f1a3db82 namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: 825835ec-b395-49ba-9a84-394073651fc8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:43:04.610535Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4 namespace: kserve-ci-e2e-test resourceVersion: "14179" uid: 15b10f19-0c65-4064-b0d7-7213aa008b0b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-669d8d6456-h8dq4 to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T15:43:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:43:04Z" name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4.18b15073757b83f7 namespace: kserve-ci-e2e-test resourceVersion: "14186" uid: b902666c-6b81-479d-b6e2-13fe400377c1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4 namespace: kserve-ci-e2e-test resourceVersion: "14196" uid: 15b10f19-0c65-4064-b0d7-7213aa008b0b kind: Event lastTimestamp: "2026-05-20T15:43:05Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:43:05Z" name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4.18b150738f6234bf namespace: kserve-ci-e2e-test resourceVersion: "14226" uid: d90f5af9-bbfa-470c-9a8a-a82d595ee2fe reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4 namespace: kserve-ci-e2e-test resourceVersion: "14181" uid: 15b10f19-0c65-4064-b0d7-7213aa008b0b kind: Event lastTimestamp: "2026-05-20T15:43:05Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:050b18baea3a969ddc2a15671f898a5f40ff53eb89f54c18856b130dc10bd894" metadata: creationTimestamp: "2026-05-20T15:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:05Z" name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4.18b1507390e14100 namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: cd8deda4-14b8-478e-864a-611e798af510 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4 namespace: kserve-ci-e2e-test resourceVersion: "14181" uid: 15b10f19-0c65-4064-b0d7-7213aa008b0b kind: Event lastTimestamp: "2026-05-20T15:43:10Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:050b18baea3a969ddc2a15671f898a5f40ff53eb89f54c18856b130dc10bd894" in 5.503s (5.503s including waiting). Image size: 301471312 bytes.' metadata: creationTimestamp: "2026-05-20T15:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:10Z" name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4.18b15074d8f0de60 namespace: kserve-ci-e2e-test resourceVersion: "14334" uid: 4388909c-d0f6-46ef-8429-4c18167dc974 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4 namespace: kserve-ci-e2e-test resourceVersion: "14181" uid: 15b10f19-0c65-4064-b0d7-7213aa008b0b kind: Event lastTimestamp: "2026-05-20T15:43:10Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T15:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:10Z" name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4.18b15074dd54696f namespace: kserve-ci-e2e-test resourceVersion: "14336" uid: d6c7a1c5-adb9-4399-b9eb-619277256c30 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4 namespace: kserve-ci-e2e-test resourceVersion: "14181" uid: 15b10f19-0c65-4064-b0d7-7213aa008b0b kind: Event lastTimestamp: "2026-05-20T15:43:10Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T15:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:10Z" name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4.18b15074ddf90f58 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: d20373a0-dc9d-499d-8634-10f51c5b459f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4 namespace: kserve-ci-e2e-test resourceVersion: "14181" uid: 15b10f19-0c65-4064-b0d7-7213aa008b0b kind: Event lastTimestamp: "2026-05-20T15:43:15Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-05-20T15:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:15Z" name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4.18b150761b36243b namespace: kserve-ci-e2e-test resourceVersion: "14390" uid: b61e807c-013e-4b5d-85a2-b8b3cd5bd77f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4 namespace: kserve-ci-e2e-test resourceVersion: "14181" uid: 15b10f19-0c65-4064-b0d7-7213aa008b0b kind: Event lastTimestamp: "2026-05-20T15:43:40Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 24.493s (24.493s including waiting). Image size: 1306444539 bytes.' metadata: creationTimestamp: "2026-05-20T15:43:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:40Z" name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4.18b1507bcf259c4e namespace: kserve-ci-e2e-test resourceVersion: "14640" uid: 69e86dd7-1b0a-42bf-bfbf-1dbaa1e17b51 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4 namespace: kserve-ci-e2e-test resourceVersion: "14181" uid: 15b10f19-0c65-4064-b0d7-7213aa008b0b kind: Event lastTimestamp: "2026-05-20T15:43:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:43:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:40Z" name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4.18b1507bd558d278 namespace: kserve-ci-e2e-test resourceVersion: "14642" uid: 983171e6-b39f-48e4-a128-db11136b9de3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4 namespace: kserve-ci-e2e-test resourceVersion: "14181" uid: 15b10f19-0c65-4064-b0d7-7213aa008b0b kind: Event lastTimestamp: "2026-05-20T15:43:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:43:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:40Z" name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4.18b1507bd68b9b5a namespace: kserve-ci-e2e-test resourceVersion: "14643" uid: 8b129da4-596a-4817-8e35-989f0dd8b267 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4 namespace: kserve-ci-e2e-test resourceVersion: "14181" uid: 15b10f19-0c65-4064-b0d7-7213aa008b0b kind: Event lastTimestamp: "2026-05-20T15:43:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T15:43:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:40Z" name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4.18b1507bd6995396 namespace: kserve-ci-e2e-test resourceVersion: "14644" uid: 18196334-b0e5-40c6-821c-9b9d11ceda00 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4 namespace: kserve-ci-e2e-test resourceVersion: "14181" uid: 15b10f19-0c65-4064-b0d7-7213aa008b0b kind: Event lastTimestamp: "2026-05-20T15:43:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:43:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:40Z" name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4.18b1507bdc776c8c namespace: kserve-ci-e2e-test resourceVersion: "14645" uid: 5621db99-629d-44d9-9bfe-24db101650ca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4 namespace: kserve-ci-e2e-test resourceVersion: "14181" uid: 15b10f19-0c65-4064-b0d7-7213aa008b0b kind: Event lastTimestamp: "2026-05-20T15:43:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:43:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:40Z" name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4.18b1507bdd9e4908 namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: 3f221e39-7a4a-45f9-a50e-8b2989f76758 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-20T15:43:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4 namespace: kserve-ci-e2e-test resourceVersion: "14181" uid: 15b10f19-0c65-4064-b0d7-7213aa008b0b kind: Event lastTimestamp: "2026-05-20T15:45:18Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:43:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:18Z" name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4.18b1507c3038d675 namespace: kserve-ci-e2e-test resourceVersion: "15877" uid: 4c49bb26-3fd9-4db3-8388-62143935ecd5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4 namespace: kserve-ci-e2e-test resourceVersion: "14181" uid: 15b10f19-0c65-4064-b0d7-7213aa008b0b kind: Event lastTimestamp: "2026-05-20T15:45:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T15:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:14Z" name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4.18b15091bf4c4b0c namespace: kserve-ci-e2e-test resourceVersion: "15798" uid: a684b90b-1889-479d-bb4e-9c6b30ad2b51 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4 namespace: kserve-ci-e2e-test resourceVersion: "14181" uid: 15b10f19-0c65-4064-b0d7-7213aa008b0b kind: Event lastTimestamp: "2026-05-20T15:45:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:14Z" name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4.18b15091bf4e6c7d namespace: kserve-ci-e2e-test resourceVersion: "15801" uid: af92cc71-b194-4768-9a95-2048ced3ba65 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4 namespace: kserve-ci-e2e-test resourceVersion: "14181" uid: 15b10f19-0c65-4064-b0d7-7213aa008b0b kind: Event lastTimestamp: "2026-05-20T15:45:18Z" message: 'Readiness probe failed: Get "https://10.132.0.36:8643/healthz": dial tcp 10.132.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:45:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:18Z" name: isvc-xgboost-graph-predictor-669d8d6456-h8dq4.18b150928a6df738 namespace: kserve-ci-e2e-test resourceVersion: "15876" uid: 154cef19-f8ee-4c07-b896-b87b19155ecc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-669d8d6456 namespace: kserve-ci-e2e-test resourceVersion: "14168" uid: ea5b632c-c730-4274-9419-bbefb7745123 kind: Event lastTimestamp: "2026-05-20T15:43:04Z" message: 'Created pod: isvc-xgboost-graph-predictor-669d8d6456-h8dq4' metadata: creationTimestamp: "2026-05-20T15:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:43:04Z" name: isvc-xgboost-graph-predictor-669d8d6456.18b15073751f1088 namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 29b450e4-9c1b-4f40-b6d9-923aecbe136f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: c84c3acf-ece2-4398-9e29-bab6dbe15e60 kind: Event lastTimestamp: "2026-05-20T15:43:04Z" message: Scaled up replica set isvc-xgboost-graph-predictor-669d8d6456 from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:43:04Z" name: isvc-xgboost-graph-predictor.18b1507370f997a6 namespace: kserve-ci-e2e-test resourceVersion: "14172" uid: 45d4f452-2af8-40c8-88a0-0c8e0c5bec81 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:43:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14166" uid: 5f119851-e731-4ab8-a7e4-428059bad53d kind: Event lastTimestamp: "2026-05-20T15:44:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:04Z" name: isvc-xgboost-graph-predictor.18b15076ef95e064 namespace: kserve-ci-e2e-test resourceVersion: "14846" uid: 77645944-7ddd-410f-b3cd-ddd5d0870344 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:43:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14166" uid: 5f119851-e731-4ab8-a7e4-428059bad53d kind: Event lastTimestamp: "2026-05-20T15:44:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:04Z" name: isvc-xgboost-graph-predictor.18b15076f0374e28 namespace: kserve-ci-e2e-test resourceVersion: "14847" uid: 3e2b9a77-c7d2-4f59-b0f6-8ceacada2b21 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:44:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 5f119851-e731-4ab8-a7e4-428059bad53d kind: Event lastTimestamp: "2026-05-20T15:45:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:04Z" name: isvc-xgboost-graph-predictor.18b15084ed48257f namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: 0b80e041-1149-4292-8f26-dc248370f3d4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:44:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: 5f119851-e731-4ab8-a7e4-428059bad53d kind: Event lastTimestamp: "2026-05-20T15:45:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:04Z" name: isvc-xgboost-graph-predictor.18b15084edcc4b2f namespace: kserve-ci-e2e-test resourceVersion: "15590" uid: efb3d349-d18a-47a2-a176-5ee8a547b81b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-20T15:43:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14151" uid: 5b1cf4bd-cf69-4b1d-9d9b-c104ee69c734 kind: Event lastTimestamp: "2026-05-20T15:44:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:44:48Z" name: isvc-xgboost-graph.18b1507370ee3367 namespace: kserve-ci-e2e-test resourceVersion: "15430" uid: be7d1e27-953b-4470-9379-760262e32bb4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15424" uid: 5b1cf4bd-cf69-4b1d-9d9b-c104ee69c734 kind: Event lastTimestamp: "2026-05-20T15:44:48Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-05-20T15:44:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:44:48Z" name: isvc-xgboost-graph.18b1508b9ae23799 namespace: kserve-ci-e2e-test resourceVersion: "15425" uid: 61a123d2-97cc-4984-84a1-27092d98f709 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:45:04.296076Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-6b9459f84c-nwvpp namespace: kserve-ci-e2e-test resourceVersion: "15566" uid: fb774bd5-e22c-4742-9d65-794519951feb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-6b9459f84c-nwvpp to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T15:45:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:45:04Z" name: model-chainer-6b9459f84c-nwvpp.18b1508f534bef63 namespace: kserve-ci-e2e-test resourceVersion: "15571" uid: 207b2a09-267c-4ba6-8f2a-79daa5db633a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:04Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-6b9459f84c-nwvpp namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: fb774bd5-e22c-4742-9d65-794519951feb kind: Event lastTimestamp: "2026-05-20T15:45:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T15:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:04Z" name: model-chainer-6b9459f84c-nwvpp.18b1508f5a1713b7 namespace: kserve-ci-e2e-test resourceVersion: "15581" uid: e61efea8-e48a-4379-be4c-ff7d7125b187 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:05Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-6b9459f84c-nwvpp namespace: kserve-ci-e2e-test resourceVersion: "15575" uid: fb774bd5-e22c-4742-9d65-794519951feb kind: Event lastTimestamp: "2026-05-20T15:45:05Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:45:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:45:05Z" name: model-chainer-6b9459f84c-nwvpp.18b1508f911f5153 namespace: kserve-ci-e2e-test resourceVersion: "15598" uid: ae2b329a-54ac-4d7b-901a-e68ce1aab4f6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-6b9459f84c-nwvpp namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: fb774bd5-e22c-4742-9d65-794519951feb kind: Event lastTimestamp: "2026-05-20T15:45:05Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:88955c235550cb8e8a9cbe850692d2683b6dea68a08743540305f46e732a59a3" already present on machine metadata: creationTimestamp: "2026-05-20T15:45:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:05Z" name: model-chainer-6b9459f84c-nwvpp.18b1508f92b8e4ea namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: c49a17a6-344d-44a7-ba68-edc2b5bc2c03 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-6b9459f84c-nwvpp namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: fb774bd5-e22c-4742-9d65-794519951feb kind: Event lastTimestamp: "2026-05-20T15:45:05Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-05-20T15:45:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:05Z" name: model-chainer-6b9459f84c-nwvpp.18b1508f9609b882 namespace: kserve-ci-e2e-test resourceVersion: "15602" uid: b76772b3-5508-446b-be12-c25cf866421b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-6b9459f84c-nwvpp namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: fb774bd5-e22c-4742-9d65-794519951feb kind: Event lastTimestamp: "2026-05-20T15:45:05Z" message: Started container model-chainer metadata: creationTimestamp: "2026-05-20T15:45:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:05Z" name: model-chainer-6b9459f84c-nwvpp.18b1508f96792b7b namespace: kserve-ci-e2e-test resourceVersion: "15604" uid: 842e1419-3ff1-4ac6-be6f-373b18da94d0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-6b9459f84c-nwvpp namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: fb774bd5-e22c-4742-9d65-794519951feb kind: Event lastTimestamp: "2026-05-20T15:45:14Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-05-20T15:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:14Z" name: model-chainer-6b9459f84c-nwvpp.18b15091aabbbea0 namespace: kserve-ci-e2e-test resourceVersion: "15699" uid: 0943b1b6-e7da-4961-a754-b3642a89eb1b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T15:45:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-6b9459f84c-nwvpp namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: fb774bd5-e22c-4742-9d65-794519951feb kind: Event lastTimestamp: "2026-05-20T15:45:42Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T15:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:42Z" name: model-chainer-6b9459f84c-nwvpp.18b15092628c24aa namespace: kserve-ci-e2e-test resourceVersion: "16095" uid: f0b4281a-d315-4175-8244-6d3cb6e8bf30 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-6b9459f84c namespace: kserve-ci-e2e-test resourceVersion: "15562" uid: 3ba00f04-b760-4e2e-b791-027a2e503de2 kind: Event lastTimestamp: "2026-05-20T15:45:04Z" message: 'Created pod: model-chainer-6b9459f84c-nwvpp' metadata: creationTimestamp: "2026-05-20T15:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:04Z" name: model-chainer-6b9459f84c.18b1508f5277bd33 namespace: kserve-ci-e2e-test resourceVersion: "15569" uid: 1e467743-6650-4895-af05-fe1a2a699eaa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15559" uid: 808a6c69-8b89-4d12-b336-b9c72cc8a35e kind: Event lastTimestamp: "2026-05-20T15:45:04Z" message: Scaled up replica set model-chainer-6b9459f84c from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:04Z" name: model-chainer.18b1508f50705de4 namespace: kserve-ci-e2e-test resourceVersion: "15564" uid: 66762ac1-ee62-4b12-9447-ace99b5c20df reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15578" uid: 08c816f5-8afd-47e6-86ce-17bf6aa22ec4 kind: Event lastTimestamp: "2026-05-20T15:45:04Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-05-20T15:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:45:04Z" name: model-chainer.18b1508f59393ebd namespace: kserve-ci-e2e-test resourceVersion: "15580" uid: 23c9cf24-91ba-476f-91aa-be81f674aeec reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:46:14.548306Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-23504-54f4b4d8bc-pqd29 namespace: kserve-ci-e2e-test resourceVersion: "16413" uid: 7d23dde2-c954-42ec-84b2-2686d2e578ad kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-23504-54f4b4d8bc-pqd29 to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T15:46:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:46:14Z" name: sequence-graph-23504-54f4b4d8bc-pqd29.18b1509faea7da53 namespace: kserve-ci-e2e-test resourceVersion: "16418" uid: bba59181-d6e5-4712-80fc-acddbbb2f2ee reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:15Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-23504-54f4b4d8bc-pqd29 namespace: kserve-ci-e2e-test resourceVersion: "16422" uid: 7d23dde2-c954-42ec-84b2-2686d2e578ad kind: Event lastTimestamp: "2026-05-20T15:46:15Z" message: Add eth0 [10.132.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:46:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:46:15Z" name: sequence-graph-23504-54f4b4d8bc-pqd29.18b1509fc9e81359 namespace: kserve-ci-e2e-test resourceVersion: "16438" uid: 7f933902-d714-472d-b6bb-1db06c3b4d67 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-23504} kind: Pod name: sequence-graph-23504-54f4b4d8bc-pqd29 namespace: kserve-ci-e2e-test resourceVersion: "16415" uid: 7d23dde2-c954-42ec-84b2-2686d2e578ad kind: Event lastTimestamp: "2026-05-20T15:46:15Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:88955c235550cb8e8a9cbe850692d2683b6dea68a08743540305f46e732a59a3" already present on machine metadata: creationTimestamp: "2026-05-20T15:46:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:46:15Z" name: sequence-graph-23504-54f4b4d8bc-pqd29.18b1509fcb46b36f namespace: kserve-ci-e2e-test resourceVersion: "16440" uid: 74267a5f-2b1e-43ea-a706-056b16b26a38 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-23504} kind: Pod name: sequence-graph-23504-54f4b4d8bc-pqd29 namespace: kserve-ci-e2e-test resourceVersion: "16415" uid: 7d23dde2-c954-42ec-84b2-2686d2e578ad kind: Event lastTimestamp: "2026-05-20T15:46:15Z" message: 'Created container: sequence-graph-23504' metadata: creationTimestamp: "2026-05-20T15:46:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:46:15Z" name: sequence-graph-23504-54f4b4d8bc-pqd29.18b1509fce9521c6 namespace: kserve-ci-e2e-test resourceVersion: "16441" uid: 05f8c21e-a555-4f28-bae2-f4fe17950971 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-23504} kind: Pod name: sequence-graph-23504-54f4b4d8bc-pqd29 namespace: kserve-ci-e2e-test resourceVersion: "16415" uid: 7d23dde2-c954-42ec-84b2-2686d2e578ad kind: Event lastTimestamp: "2026-05-20T15:46:15Z" message: Started container sequence-graph-23504 metadata: creationTimestamp: "2026-05-20T15:46:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:46:15Z" name: sequence-graph-23504-54f4b4d8bc-pqd29.18b1509fcf1555fc namespace: kserve-ci-e2e-test resourceVersion: "16442" uid: 1503f9a3-9f89-40b3-b239-64fce2012a6c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-23504} kind: Pod name: sequence-graph-23504-54f4b4d8bc-pqd29 namespace: kserve-ci-e2e-test resourceVersion: "16415" uid: 7d23dde2-c954-42ec-84b2-2686d2e578ad kind: Event lastTimestamp: "2026-05-20T15:54:29Z" message: Stopping container sequence-graph-23504 metadata: creationTimestamp: "2026-05-20T15:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:29Z" name: sequence-graph-23504-54f4b4d8bc-pqd29.18b15112e0d075d2 namespace: kserve-ci-e2e-test resourceVersion: "20001" uid: 0a955b21-42d3-4dde-be21-fda459042a9f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T15:54:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-23504} kind: Pod name: sequence-graph-23504-54f4b4d8bc-pqd29 namespace: kserve-ci-e2e-test resourceVersion: "16415" uid: 7d23dde2-c954-42ec-84b2-2686d2e578ad kind: Event lastTimestamp: "2026-05-20T15:54:56Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T15:54:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:56Z" name: sequence-graph-23504-54f4b4d8bc-pqd29.18b151136f2612dd namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: a92de6f7-8152-472d-8a46-96a113ac3ecc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-23504-54f4b4d8bc namespace: kserve-ci-e2e-test resourceVersion: "16410" uid: 045c4e1b-00a9-48f7-9d35-f2258a59c534 kind: Event lastTimestamp: "2026-05-20T15:46:14Z" message: 'Created pod: sequence-graph-23504-54f4b4d8bc-pqd29' metadata: creationTimestamp: "2026-05-20T15:46:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:46:14Z" name: sequence-graph-23504-54f4b4d8bc.18b1509fae3d42f6 namespace: kserve-ci-e2e-test resourceVersion: "16414" uid: f29733b9-d28e-466e-b369-563d3bca343b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-23504 namespace: kserve-ci-e2e-test resourceVersion: "16408" uid: f636bd81-5650-4904-b360-7e572bfdc8c9 kind: Event lastTimestamp: "2026-05-20T15:46:14Z" message: Scaled up replica set sequence-graph-23504-54f4b4d8bc from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:46:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:46:14Z" name: sequence-graph-23504.18b1509fad9998d4 namespace: kserve-ci-e2e-test resourceVersion: "16411" uid: 2d5f50a3-dbda-4a5b-8179-7dc599b90d33 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:14Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-23504 namespace: kserve-ci-e2e-test resourceVersion: "16427" uid: 96134d2b-f64b-4ec7-8819-7fa1b0eb34c7 kind: Event lastTimestamp: "2026-05-20T15:46:14Z" message: InferenceGraph [sequence-graph-23504] is Ready metadata: creationTimestamp: "2026-05-20T15:46:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:46:14Z" name: sequence-graph-23504.18b1509fb1f62d14 namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: bc5c41ab-4fea-4581-b7a7-3d27ae6e3a39 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:46:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-23504 namespace: kserve-ci-e2e-test resourceVersion: "16409" uid: 47efa703-46a8-4b13-ac86-9cf15698f4ea kind: Event lastTimestamp: "2026-05-20T15:46:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:46:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:46:59Z" name: sequence-graph-23504.18b150a32c4dbd9f namespace: kserve-ci-e2e-test resourceVersion: "16769" uid: e962619e-e488-4960-b733-b3176b176c97 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:46:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-23504 namespace: kserve-ci-e2e-test resourceVersion: "16409" uid: 47efa703-46a8-4b13-ac86-9cf15698f4ea kind: Event lastTimestamp: "2026-05-20T15:46:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:46:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:46:59Z" name: sequence-graph-23504.18b150a32cae00f4 namespace: kserve-ci-e2e-test resourceVersion: "16770" uid: 7046b3b7-6d30-49e4-add5-168e8725cdb4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:55:29.484455Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-4ff76-59c94c4c8c-4gzqz namespace: kserve-ci-e2e-test resourceVersion: "20844" uid: 763038c5-1d94-4e7e-88bb-a730362c7469 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-4ff76-59c94c4c8c-4gzqz to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T15:55:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:55:29Z" name: sequence-graph-4ff76-59c94c4c8c-4gzqz.18b15120e36d8c68 namespace: kserve-ci-e2e-test resourceVersion: "20849" uid: 7dd4eb0e-e669-45fb-8f87-8f851bb3d1af reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:29Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-4ff76-59c94c4c8c-4gzqz namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: 763038c5-1d94-4e7e-88bb-a730362c7469 kind: Event lastTimestamp: "2026-05-20T15:55:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-4ff76-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T15:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:29Z" name: sequence-graph-4ff76-59c94c4c8c-4gzqz.18b15120f3024178 namespace: kserve-ci-e2e-test resourceVersion: "20866" uid: c78a66ac-ef9b-4ff6-a0c8-f3d1c9e559cc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:30Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-4ff76-59c94c4c8c-4gzqz namespace: kserve-ci-e2e-test resourceVersion: "20853" uid: 763038c5-1d94-4e7e-88bb-a730362c7469 kind: Event lastTimestamp: "2026-05-20T15:55:30Z" message: Add eth0 [10.132.0.53/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:55:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:55:30Z" name: sequence-graph-4ff76-59c94c4c8c-4gzqz.18b15121212ea8b8 namespace: kserve-ci-e2e-test resourceVersion: "20874" uid: af95f0ea-1294-4ee0-8468-eba1e254a8fc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-4ff76} kind: Pod name: sequence-graph-4ff76-59c94c4c8c-4gzqz namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: 763038c5-1d94-4e7e-88bb-a730362c7469 kind: Event lastTimestamp: "2026-05-20T15:55:30Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:88955c235550cb8e8a9cbe850692d2683b6dea68a08743540305f46e732a59a3" already present on machine metadata: creationTimestamp: "2026-05-20T15:55:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:30Z" name: sequence-graph-4ff76-59c94c4c8c-4gzqz.18b15121229ed7a2 namespace: kserve-ci-e2e-test resourceVersion: "20877" uid: 25dcc9c6-087b-4b0a-899c-7cf5305bbb66 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-4ff76} kind: Pod name: sequence-graph-4ff76-59c94c4c8c-4gzqz namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: 763038c5-1d94-4e7e-88bb-a730362c7469 kind: Event lastTimestamp: "2026-05-20T15:55:30Z" message: 'Created container: sequence-graph-4ff76' metadata: creationTimestamp: "2026-05-20T15:55:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:30Z" name: sequence-graph-4ff76-59c94c4c8c-4gzqz.18b151212694f5da namespace: kserve-ci-e2e-test resourceVersion: "20878" uid: c79073d9-637d-494b-9601-5e32542c8595 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-4ff76} kind: Pod name: sequence-graph-4ff76-59c94c4c8c-4gzqz namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: 763038c5-1d94-4e7e-88bb-a730362c7469 kind: Event lastTimestamp: "2026-05-20T15:55:30Z" message: Started container sequence-graph-4ff76 metadata: creationTimestamp: "2026-05-20T15:55:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:30Z" name: sequence-graph-4ff76-59c94c4c8c-4gzqz.18b1512127219e01 namespace: kserve-ci-e2e-test resourceVersion: "20879" uid: 7b4f9aad-c28b-4f4a-a83c-7ffbb6fd4b13 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-4ff76} kind: Pod name: sequence-graph-4ff76-59c94c4c8c-4gzqz namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: 763038c5-1d94-4e7e-88bb-a730362c7469 kind: Event lastTimestamp: "2026-05-20T15:55:39Z" message: Stopping container sequence-graph-4ff76 metadata: creationTimestamp: "2026-05-20T15:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:39Z" name: sequence-graph-4ff76-59c94c4c8c-4gzqz.18b151233cd36680 namespace: kserve-ci-e2e-test resourceVersion: "20979" uid: 9109d4c4-89ab-4792-b5bb-e2e53084925e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T15:55:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-4ff76} kind: Pod name: sequence-graph-4ff76-59c94c4c8c-4gzqz namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: 763038c5-1d94-4e7e-88bb-a730362c7469 kind: Event lastTimestamp: "2026-05-20T15:56:06Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T15:55:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:56:06Z" name: sequence-graph-4ff76-59c94c4c8c-4gzqz.18b15123bcee4aeb namespace: kserve-ci-e2e-test resourceVersion: "21324" uid: 1a10928b-a9c2-4199-8dc6-a5ca07d17843 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-4ff76-59c94c4c8c namespace: kserve-ci-e2e-test resourceVersion: "20841" uid: 43cab018-b07a-458f-a422-3137237e3a27 kind: Event lastTimestamp: "2026-05-20T15:55:29Z" message: 'Created pod: sequence-graph-4ff76-59c94c4c8c-4gzqz' metadata: creationTimestamp: "2026-05-20T15:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:55:29Z" name: sequence-graph-4ff76-59c94c4c8c.18b15120e301c3f5 namespace: kserve-ci-e2e-test resourceVersion: "20845" uid: 08c4010b-efe9-4d73-92bf-dc81ba580b56 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-4ff76 namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: c9f0b48b-d79e-4616-9b0e-cce7446dc84e kind: Event lastTimestamp: "2026-05-20T15:55:29Z" message: Scaled up replica set sequence-graph-4ff76-59c94c4c8c from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:55:29Z" name: sequence-graph-4ff76.18b15120e25ed06e namespace: kserve-ci-e2e-test resourceVersion: "20842" uid: c749da90-98a5-440a-8362-019676304540 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:29Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-4ff76 namespace: kserve-ci-e2e-test resourceVersion: "20857" uid: 6bb4a9bd-4758-4ac5-b2a8-93fe8fcb449e kind: Event lastTimestamp: "2026-05-20T15:55:29Z" message: InferenceGraph [sequence-graph-4ff76] is Ready metadata: creationTimestamp: "2026-05-20T15:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:55:29Z" name: sequence-graph-4ff76.18b15120e6e1abc3 namespace: kserve-ci-e2e-test resourceVersion: "20860" uid: 98409b48-bb9a-42be-8315-059bd285cacd reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:56:49.802275Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-80b6e-cd77b6b8-n7mpf namespace: kserve-ci-e2e-test resourceVersion: "21745" uid: 33fa91a6-9459-488a-b32d-52f6bbdd4fcc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-80b6e-cd77b6b8-n7mpf to ip-10-0-135-174.ec2.internal metadata: creationTimestamp: "2026-05-20T15:56:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:56:49Z" name: sequence-graph-80b6e-cd77b6b8-n7mpf.18b1513396be2cfc namespace: kserve-ci-e2e-test resourceVersion: "21750" uid: 26238718-bd6c-482d-88a5-0dab2a354bf0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:56:50Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-80b6e-cd77b6b8-n7mpf namespace: kserve-ci-e2e-test resourceVersion: "21747" uid: 33fa91a6-9459-488a-b32d-52f6bbdd4fcc kind: Event lastTimestamp: "2026-05-20T15:56:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-80b6e-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T15:56:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:56:50Z" name: sequence-graph-80b6e-cd77b6b8-n7mpf.18b15133a681c721 namespace: kserve-ci-e2e-test resourceVersion: "21762" uid: 88e0681e-3166-4411-851a-1090706aaf3e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:56:50Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-80b6e-cd77b6b8-n7mpf namespace: kserve-ci-e2e-test resourceVersion: "21754" uid: 33fa91a6-9459-488a-b32d-52f6bbdd4fcc kind: Event lastTimestamp: "2026-05-20T15:56:50Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:56:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:56:50Z" name: sequence-graph-80b6e-cd77b6b8-n7mpf.18b15133d40e8b0f namespace: kserve-ci-e2e-test resourceVersion: "21772" uid: bc2129ca-1d69-4c66-9220-0a0a6d12265d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:56:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-80b6e} kind: Pod name: sequence-graph-80b6e-cd77b6b8-n7mpf namespace: kserve-ci-e2e-test resourceVersion: "21747" uid: 33fa91a6-9459-488a-b32d-52f6bbdd4fcc kind: Event lastTimestamp: "2026-05-20T15:56:50Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:88955c235550cb8e8a9cbe850692d2683b6dea68a08743540305f46e732a59a3" metadata: creationTimestamp: "2026-05-20T15:56:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:56:50Z" name: sequence-graph-80b6e-cd77b6b8-n7mpf.18b15133d55c5ffb namespace: kserve-ci-e2e-test resourceVersion: "21774" uid: 6347d5a0-5b93-4f95-9d3e-3a2aaf875afc reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:56:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-80b6e} kind: Pod name: sequence-graph-80b6e-cd77b6b8-n7mpf namespace: kserve-ci-e2e-test resourceVersion: "21747" uid: 33fa91a6-9459-488a-b32d-52f6bbdd4fcc kind: Event lastTimestamp: "2026-05-20T15:56:52Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:88955c235550cb8e8a9cbe850692d2683b6dea68a08743540305f46e732a59a3" in 2.13s (2.13s including waiting). Image size: 218002168 bytes.' metadata: creationTimestamp: "2026-05-20T15:56:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:56:52Z" name: sequence-graph-80b6e-cd77b6b8-n7mpf.18b15134546005de namespace: kserve-ci-e2e-test resourceVersion: "21793" uid: 3eb7fb5d-5614-4c35-8071-083ac60c2530 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:56:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-80b6e} kind: Pod name: sequence-graph-80b6e-cd77b6b8-n7mpf namespace: kserve-ci-e2e-test resourceVersion: "21747" uid: 33fa91a6-9459-488a-b32d-52f6bbdd4fcc kind: Event lastTimestamp: "2026-05-20T15:56:53Z" message: 'Created container: sequence-graph-80b6e' metadata: creationTimestamp: "2026-05-20T15:56:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:56:53Z" name: sequence-graph-80b6e-cd77b6b8-n7mpf.18b151345758d21a namespace: kserve-ci-e2e-test resourceVersion: "21794" uid: db2b13d1-3617-4355-90cd-b98793e3e4d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:56:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-80b6e} kind: Pod name: sequence-graph-80b6e-cd77b6b8-n7mpf namespace: kserve-ci-e2e-test resourceVersion: "21747" uid: 33fa91a6-9459-488a-b32d-52f6bbdd4fcc kind: Event lastTimestamp: "2026-05-20T15:56:53Z" message: Started container sequence-graph-80b6e metadata: creationTimestamp: "2026-05-20T15:56:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:56:53Z" name: sequence-graph-80b6e-cd77b6b8-n7mpf.18b1513457ce17a0 namespace: kserve-ci-e2e-test resourceVersion: "21795" uid: 16ec8989-9548-43a0-b8f1-449969dc414c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-80b6e} kind: Pod name: sequence-graph-80b6e-cd77b6b8-n7mpf namespace: kserve-ci-e2e-test resourceVersion: "21747" uid: 33fa91a6-9459-488a-b32d-52f6bbdd4fcc kind: Event lastTimestamp: "2026-05-20T16:05:04Z" message: Stopping container sequence-graph-80b6e metadata: creationTimestamp: "2026-05-20T16:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:04Z" name: sequence-graph-80b6e-cd77b6b8-n7mpf.18b151a6c8914b8b namespace: kserve-ci-e2e-test resourceVersion: "25603" uid: 5a1517c6-8b41-4055-95c3-5eec06577fb5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T16:05:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-80b6e} kind: Pod name: sequence-graph-80b6e-cd77b6b8-n7mpf namespace: kserve-ci-e2e-test resourceVersion: "21747" uid: 33fa91a6-9459-488a-b32d-52f6bbdd4fcc kind: Event lastTimestamp: "2026-05-20T16:05:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T16:05:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:30Z" name: sequence-graph-80b6e-cd77b6b8-n7mpf.18b151a6e34c2597 namespace: kserve-ci-e2e-test resourceVersion: "25948" uid: 42af7ba5-6e95-4bf2-bee7-07743f883d23 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:56:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-80b6e-cd77b6b8 namespace: kserve-ci-e2e-test resourceVersion: "21742" uid: bf251dee-7b64-452f-ac0e-a9818e5043fe kind: Event lastTimestamp: "2026-05-20T15:56:49Z" message: 'Created pod: sequence-graph-80b6e-cd77b6b8-n7mpf' metadata: creationTimestamp: "2026-05-20T15:56:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:56:49Z" name: sequence-graph-80b6e-cd77b6b8.18b15133963f9396 namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: c33915ba-9d98-49dd-a972-cab9239f68a2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:56:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-80b6e namespace: kserve-ci-e2e-test resourceVersion: "21740" uid: ab92c5fc-6fc4-4054-a68d-b2b228b09b15 kind: Event lastTimestamp: "2026-05-20T15:56:49Z" message: Scaled up replica set sequence-graph-80b6e-cd77b6b8 from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:56:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:56:49Z" name: sequence-graph-80b6e.18b151339591625c namespace: kserve-ci-e2e-test resourceVersion: "21743" uid: e8297b48-4543-48bc-99b7-85e5578735e0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:56:49Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-80b6e namespace: kserve-ci-e2e-test resourceVersion: "21756" uid: b793fdfd-bbb3-42d6-9a5c-6ea45d00218e kind: Event lastTimestamp: "2026-05-20T15:56:49Z" message: InferenceGraph [sequence-graph-80b6e] is Ready metadata: creationTimestamp: "2026-05-20T15:56:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:56:49Z" name: sequence-graph-80b6e.18b151339a05c6d0 namespace: kserve-ci-e2e-test resourceVersion: "21758" uid: 66295917-4180-4962-90cb-e6172e37714e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:57:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-80b6e namespace: kserve-ci-e2e-test resourceVersion: "21741" uid: 505f31a0-e6e7-4e20-b4d9-b5a3138b90dc kind: Event lastTimestamp: "2026-05-20T15:57:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:57:34Z" name: sequence-graph-80b6e.18b15137141f40f4 namespace: kserve-ci-e2e-test resourceVersion: "22102" uid: 2c3c3cf9-75f9-4187-9ec2-7d3fe489a1e8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:57:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-80b6e namespace: kserve-ci-e2e-test resourceVersion: "21741" uid: 505f31a0-e6e7-4e20-b4d9-b5a3138b90dc kind: Event lastTimestamp: "2026-05-20T15:57:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:57:34Z" name: sequence-graph-80b6e.18b15137148595e4 namespace: kserve-ci-e2e-test resourceVersion: "22103" uid: 2c2f9bae-7323-4438-b03d-00e27aaef0d6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-20T16:07:09.420935Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-62c44-6586f8b5bc-lm5g9 namespace: kserve-ci-e2e-test resourceVersion: "27020" uid: 327197d7-c7e5-4bf7-a815-b263d14b655f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-62c44-6586f8b5bc-lm5g9 to ip-10-0-135-174.ec2.internal metadata: creationTimestamp: "2026-05-20T16:07:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T16:07:09Z" name: splitter-graph-62c44-6586f8b5bc-lm5g9.18b151c3dae4e19c namespace: kserve-ci-e2e-test resourceVersion: "27026" uid: 7284af47-ec70-4c89-9e96-4cef6281f788 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:07:09Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-62c44-6586f8b5bc-lm5g9 namespace: kserve-ci-e2e-test resourceVersion: "27028" uid: 327197d7-c7e5-4bf7-a815-b263d14b655f kind: Event lastTimestamp: "2026-05-20T16:07:09Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T16:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T16:07:09Z" name: splitter-graph-62c44-6586f8b5bc-lm5g9.18b151c3f40fe6b1 namespace: kserve-ci-e2e-test resourceVersion: "27042" uid: 35ca1c95-2e2e-4a63-b6aa-37e810ee30a3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-62c44} kind: Pod name: splitter-graph-62c44-6586f8b5bc-lm5g9 namespace: kserve-ci-e2e-test resourceVersion: "27021" uid: 327197d7-c7e5-4bf7-a815-b263d14b655f kind: Event lastTimestamp: "2026-05-20T16:07:09Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:88955c235550cb8e8a9cbe850692d2683b6dea68a08743540305f46e732a59a3" already present on machine metadata: creationTimestamp: "2026-05-20T16:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:07:09Z" name: splitter-graph-62c44-6586f8b5bc-lm5g9.18b151c3f561832f namespace: kserve-ci-e2e-test resourceVersion: "27044" uid: 889983e8-c5d1-44d6-bc7e-1727ce7c4040 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-62c44} kind: Pod name: splitter-graph-62c44-6586f8b5bc-lm5g9 namespace: kserve-ci-e2e-test resourceVersion: "27021" uid: 327197d7-c7e5-4bf7-a815-b263d14b655f kind: Event lastTimestamp: "2026-05-20T16:07:09Z" message: 'Created container: splitter-graph-62c44' metadata: creationTimestamp: "2026-05-20T16:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:07:09Z" name: splitter-graph-62c44-6586f8b5bc-lm5g9.18b151c3f86bb6e4 namespace: kserve-ci-e2e-test resourceVersion: "27045" uid: 557a8ead-f2b2-4b2a-880a-bbbfe1fe1e38 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:07:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-62c44} kind: Pod name: splitter-graph-62c44-6586f8b5bc-lm5g9 namespace: kserve-ci-e2e-test resourceVersion: "27021" uid: 327197d7-c7e5-4bf7-a815-b263d14b655f kind: Event lastTimestamp: "2026-05-20T16:07:09Z" message: Started container splitter-graph-62c44 metadata: creationTimestamp: "2026-05-20T16:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:07:09Z" name: splitter-graph-62c44-6586f8b5bc-lm5g9.18b151c3f8cba04c namespace: kserve-ci-e2e-test resourceVersion: "27046" uid: 6cbbec8e-121f-4e0e-a638-c0e2f2f56671 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:15:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-62c44} kind: Pod name: splitter-graph-62c44-6586f8b5bc-lm5g9 namespace: kserve-ci-e2e-test resourceVersion: "27021" uid: 327197d7-c7e5-4bf7-a815-b263d14b655f kind: Event lastTimestamp: "2026-05-20T16:15:24Z" message: Stopping container splitter-graph-62c44 metadata: creationTimestamp: "2026-05-20T16:15:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:15:24Z" name: splitter-graph-62c44-6586f8b5bc-lm5g9.18b152370781fbc6 namespace: kserve-ci-e2e-test resourceVersion: "30468" uid: 7afc02eb-82d6-42a8-a1e6-060597050a93 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T16:15:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-62c44} kind: Pod name: splitter-graph-62c44-6586f8b5bc-lm5g9 namespace: kserve-ci-e2e-test resourceVersion: "27021" uid: 327197d7-c7e5-4bf7-a815-b263d14b655f kind: Event lastTimestamp: "2026-05-20T16:15:51Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T16:15:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:15:51Z" name: splitter-graph-62c44-6586f8b5bc-lm5g9.18b15237a86e0d15 namespace: kserve-ci-e2e-test resourceVersion: "30706" uid: fa4c3be6-d2cd-4101-91cb-60f90bc742b9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:07:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-62c44-6586f8b5bc namespace: kserve-ci-e2e-test resourceVersion: "27017" uid: 9a07b57a-76b9-4204-bf8a-f9557dfad112 kind: Event lastTimestamp: "2026-05-20T16:07:09Z" message: 'Created pod: splitter-graph-62c44-6586f8b5bc-lm5g9' metadata: creationTimestamp: "2026-05-20T16:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:07:09Z" name: splitter-graph-62c44-6586f8b5bc.18b151c3da84c2be namespace: kserve-ci-e2e-test resourceVersion: "27022" uid: 16995cae-3d33-494b-b238-b312ba22a7fd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:07:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-62c44 namespace: kserve-ci-e2e-test resourceVersion: "27015" uid: a1fe72df-1f17-4b66-80d1-3f31afac2f95 kind: Event lastTimestamp: "2026-05-20T16:07:09Z" message: Scaled up replica set splitter-graph-62c44-6586f8b5bc from 0 to 1 metadata: creationTimestamp: "2026-05-20T16:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:07:09Z" name: splitter-graph-62c44.18b151c3d9430b6b namespace: kserve-ci-e2e-test resourceVersion: "27018" uid: 88879746-d585-4306-8ae3-692a03d5f47c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:07:09Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-62c44 namespace: kserve-ci-e2e-test resourceVersion: "27033" uid: beadd5d1-4b4f-4e45-a2d8-4a8cb547360d kind: Event lastTimestamp: "2026-05-20T16:07:09Z" message: InferenceGraph [splitter-graph-62c44] is Ready metadata: creationTimestamp: "2026-05-20T16:07:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:07:09Z" name: splitter-graph-62c44.18b151c3e04087f5 namespace: kserve-ci-e2e-test resourceVersion: "27035" uid: 8a7b8376-83ff-4267-943b-6ac635dfcb32 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:07:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-62c44 namespace: kserve-ci-e2e-test resourceVersion: "27016" uid: 53f2a5ee-c5a3-4ebc-9b7b-b98d44c6dbd0 kind: Event lastTimestamp: "2026-05-20T16:07:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T16:07:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:07:24Z" name: splitter-graph-62c44.18b151c757f5cc9e namespace: kserve-ci-e2e-test resourceVersion: "27156" uid: 8153656b-a2d6-47c2-b0f7-f849bf75fb49 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:07:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-62c44 namespace: kserve-ci-e2e-test resourceVersion: "27016" uid: 53f2a5ee-c5a3-4ebc-9b7b-b98d44c6dbd0 kind: Event lastTimestamp: "2026-05-20T16:07:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T16:07:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:07:24Z" name: splitter-graph-62c44.18b151c758a5e0b3 namespace: kserve-ci-e2e-test resourceVersion: "27158" uid: b6ab4c36-5eb5-4f10-a50f-13abc1873215 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-20T16:05:59.160430Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-e05eb-5b84678cc8-ktfv6 namespace: kserve-ci-e2e-test resourceVersion: "26213" uid: a1234a3a-7afc-43d7-a3b6-fd0266f388f4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-e05eb-5b84678cc8-ktfv6 to ip-10-0-135-174.ec2.internal metadata: creationTimestamp: "2026-05-20T16:05:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T16:05:59Z" name: splitter-graph-e05eb-5b84678cc8-ktfv6.18b151b37f0a5438 namespace: kserve-ci-e2e-test resourceVersion: "26219" uid: 075bb852-9cd1-4a56-adc1-86273188bac1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:59Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-e05eb-5b84678cc8-ktfv6 namespace: kserve-ci-e2e-test resourceVersion: "26214" uid: a1234a3a-7afc-43d7-a3b6-fd0266f388f4 kind: Event lastTimestamp: "2026-05-20T16:05:59Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-e05eb-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T16:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:59Z" name: splitter-graph-e05eb-5b84678cc8-ktfv6.18b151b38af69272 namespace: kserve-ci-e2e-test resourceVersion: "26229" uid: 6b745b77-883a-42d0-897f-e3e613581c64 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:00Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-e05eb-5b84678cc8-ktfv6 namespace: kserve-ci-e2e-test resourceVersion: "26220" uid: a1234a3a-7afc-43d7-a3b6-fd0266f388f4 kind: Event lastTimestamp: "2026-05-20T16:06:00Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T16:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T16:06:00Z" name: splitter-graph-e05eb-5b84678cc8-ktfv6.18b151b3bbe0eaf1 namespace: kserve-ci-e2e-test resourceVersion: "26237" uid: 62d769b5-37bd-416b-ac29-d2269d4f3bac reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e05eb} kind: Pod name: splitter-graph-e05eb-5b84678cc8-ktfv6 namespace: kserve-ci-e2e-test resourceVersion: "26214" uid: a1234a3a-7afc-43d7-a3b6-fd0266f388f4 kind: Event lastTimestamp: "2026-05-20T16:06:00Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:88955c235550cb8e8a9cbe850692d2683b6dea68a08743540305f46e732a59a3" already present on machine metadata: creationTimestamp: "2026-05-20T16:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:00Z" name: splitter-graph-e05eb-5b84678cc8-ktfv6.18b151b3bd9dac3d namespace: kserve-ci-e2e-test resourceVersion: "26239" uid: bc7e3807-4ccc-4db6-b576-7281713851c3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e05eb} kind: Pod name: splitter-graph-e05eb-5b84678cc8-ktfv6 namespace: kserve-ci-e2e-test resourceVersion: "26214" uid: a1234a3a-7afc-43d7-a3b6-fd0266f388f4 kind: Event lastTimestamp: "2026-05-20T16:06:00Z" message: 'Created container: splitter-graph-e05eb' metadata: creationTimestamp: "2026-05-20T16:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:00Z" name: splitter-graph-e05eb-5b84678cc8-ktfv6.18b151b3c03f8fed namespace: kserve-ci-e2e-test resourceVersion: "26240" uid: 849aa843-0861-47e9-ab79-ed0f3fa3312b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e05eb} kind: Pod name: splitter-graph-e05eb-5b84678cc8-ktfv6 namespace: kserve-ci-e2e-test resourceVersion: "26214" uid: a1234a3a-7afc-43d7-a3b6-fd0266f388f4 kind: Event lastTimestamp: "2026-05-20T16:06:00Z" message: Started container splitter-graph-e05eb metadata: creationTimestamp: "2026-05-20T16:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:00Z" name: splitter-graph-e05eb-5b84678cc8-ktfv6.18b151b3c0c7f19e namespace: kserve-ci-e2e-test resourceVersion: "26241" uid: c736b686-3b5f-484c-8de6-68b7b6c1b739 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e05eb} kind: Pod name: splitter-graph-e05eb-5b84678cc8-ktfv6 namespace: kserve-ci-e2e-test resourceVersion: "26214" uid: a1234a3a-7afc-43d7-a3b6-fd0266f388f4 kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: Stopping container splitter-graph-e05eb metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:09Z" name: splitter-graph-e05eb-5b84678cc8-ktfv6.18b151b5d2a9c44e namespace: kserve-ci-e2e-test resourceVersion: "26356" uid: 515ccdd7-c770-427b-b54b-16f2748450a6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T16:06:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e05eb} kind: Pod name: splitter-graph-e05eb-5b84678cc8-ktfv6 namespace: kserve-ci-e2e-test resourceVersion: "26214" uid: a1234a3a-7afc-43d7-a3b6-fd0266f388f4 kind: Event lastTimestamp: "2026-05-20T16:06:36Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T16:06:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:36Z" name: splitter-graph-e05eb-5b84678cc8-ktfv6.18b151b66316d2f9 namespace: kserve-ci-e2e-test resourceVersion: "26749" uid: 8e63434e-c8e7-49bf-9ea1-d3aa40895b3b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-e05eb-5b84678cc8 namespace: kserve-ci-e2e-test resourceVersion: "26210" uid: dcd42096-0fd4-4b40-bc1f-d5152f410d5d kind: Event lastTimestamp: "2026-05-20T16:05:59Z" message: 'Created pod: splitter-graph-e05eb-5b84678cc8-ktfv6' metadata: creationTimestamp: "2026-05-20T16:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:05:59Z" name: splitter-graph-e05eb-5b84678cc8.18b151b37e26048b namespace: kserve-ci-e2e-test resourceVersion: "26215" uid: d46909b5-cbd6-4a2d-84d1-b7237863ca9f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-e05eb namespace: kserve-ci-e2e-test resourceVersion: "26208" uid: 7bc3038e-239a-4387-8682-dfa9e3808aee kind: Event lastTimestamp: "2026-05-20T16:05:59Z" message: Scaled up replica set splitter-graph-e05eb-5b84678cc8 from 0 to 1 metadata: creationTimestamp: "2026-05-20T16:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:05:59Z" name: splitter-graph-e05eb.18b151b37caf2194 namespace: kserve-ci-e2e-test resourceVersion: "26211" uid: dfec9192-67cb-4699-afb7-bbcb218c7352 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:59Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-e05eb namespace: kserve-ci-e2e-test resourceVersion: "26224" uid: e2d6cef9-4664-4c26-bc0a-baf83825241a kind: Event lastTimestamp: "2026-05-20T16:05:59Z" message: InferenceGraph [splitter-graph-e05eb] is Ready metadata: creationTimestamp: "2026-05-20T16:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:05:59Z" name: splitter-graph-e05eb.18b151b384e4d670 namespace: kserve-ci-e2e-test resourceVersion: "26226" uid: a5913924-ddd3-4ccc-b2b0-266fc38d1836 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:45:14.564625Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-23504-predictor-557c78fdfd-f6js7 namespace: kserve-ci-e2e-test resourceVersion: "15742" uid: 7922d8ef-19c4-4ef9-a09b-f16db6cf33e5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-23504-predictor-557c78fdfd-f6js7 to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T15:45:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:45:14Z" name: success-200-isvc-23504-predictor-557c78fdfd-f6js7.18b15091b75988a2 namespace: kserve-ci-e2e-test resourceVersion: "15751" uid: e76db091-33d6-4991-a5c3-be2da044c27c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:14Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-23504-predictor-557c78fdfd-f6js7 namespace: kserve-ci-e2e-test resourceVersion: "15745" uid: 7922d8ef-19c4-4ef9-a09b-f16db6cf33e5 kind: Event lastTimestamp: "2026-05-20T15:45:14Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-23504-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T15:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:14Z" name: success-200-isvc-23504-predictor-557c78fdfd-f6js7.18b15091bf4a6b8c namespace: kserve-ci-e2e-test resourceVersion: "15796" uid: 3ef68b7d-a581-4d3e-b09f-ba80504a5112 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:15Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-23504-predictor-557c78fdfd-f6js7 namespace: kserve-ci-e2e-test resourceVersion: "15763" uid: 7922d8ef-19c4-4ef9-a09b-f16db6cf33e5 kind: Event lastTimestamp: "2026-05-20T15:45:15Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:45:15Z" name: success-200-isvc-23504-predictor-557c78fdfd-f6js7.18b15091f5dc51ca namespace: kserve-ci-e2e-test resourceVersion: "15833" uid: 26a770ba-46c3-408a-ba34-cb3bc53b8ef9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-23504-predictor-557c78fdfd-f6js7 namespace: kserve-ci-e2e-test resourceVersion: "15745" uid: 7922d8ef-19c4-4ef9-a09b-f16db6cf33e5 kind: Event lastTimestamp: "2026-05-20T15:45:15Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-05-20T15:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:15Z" name: success-200-isvc-23504-predictor-557c78fdfd-f6js7.18b15091f755fb66 namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: af9961b5-f4fa-4aaa-a7fa-6b30c9a437be reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-23504-predictor-557c78fdfd-f6js7 namespace: kserve-ci-e2e-test resourceVersion: "15745" uid: 7922d8ef-19c4-4ef9-a09b-f16db6cf33e5 kind: Event lastTimestamp: "2026-05-20T15:45:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:15Z" name: success-200-isvc-23504-predictor-557c78fdfd-f6js7.18b15091fb89831e namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 7d9681cb-fc51-44b4-9c85-4257241971c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-23504-predictor-557c78fdfd-f6js7 namespace: kserve-ci-e2e-test resourceVersion: "15745" uid: 7922d8ef-19c4-4ef9-a09b-f16db6cf33e5 kind: Event lastTimestamp: "2026-05-20T15:45:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:15Z" name: success-200-isvc-23504-predictor-557c78fdfd-f6js7.18b15091fc1be579 namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: 0f5467ca-b07c-4f01-acc4-a4e6862d938b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-23504-predictor-557c78fdfd-f6js7 namespace: kserve-ci-e2e-test resourceVersion: "15745" uid: 7922d8ef-19c4-4ef9-a09b-f16db6cf33e5 kind: Event lastTimestamp: "2026-05-20T15:45:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T15:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:15Z" name: success-200-isvc-23504-predictor-557c78fdfd-f6js7.18b15091fc29a997 namespace: kserve-ci-e2e-test resourceVersion: "15840" uid: 4d049544-de3f-42d5-8a07-f3b8be8fe2b5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-23504-predictor-557c78fdfd-f6js7 namespace: kserve-ci-e2e-test resourceVersion: "15745" uid: 7922d8ef-19c4-4ef9-a09b-f16db6cf33e5 kind: Event lastTimestamp: "2026-05-20T15:45:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:15Z" name: success-200-isvc-23504-predictor-557c78fdfd-f6js7.18b15091ff285854 namespace: kserve-ci-e2e-test resourceVersion: "15844" uid: 6218a257-4a0d-4318-be53-e341b1fe4a92 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-23504-predictor-557c78fdfd-f6js7 namespace: kserve-ci-e2e-test resourceVersion: "15745" uid: 7922d8ef-19c4-4ef9-a09b-f16db6cf33e5 kind: Event lastTimestamp: "2026-05-20T15:45:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:45:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:15Z" name: success-200-isvc-23504-predictor-557c78fdfd-f6js7.18b15091ffc1cf75 namespace: kserve-ci-e2e-test resourceVersion: "15845" uid: df90e941-8ae9-4ef8-b3d1-36886c0235d2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-20T15:45:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-23504-predictor-557c78fdfd-f6js7 namespace: kserve-ci-e2e-test resourceVersion: "15745" uid: 7922d8ef-19c4-4ef9-a09b-f16db6cf33e5 kind: Event lastTimestamp: "2026-05-20T15:45:53Z" message: 'Readiness probe failed: dial tcp 10.132.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:45:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:53Z" name: success-200-isvc-23504-predictor-557c78fdfd-f6js7.18b15092290276c1 namespace: kserve-ci-e2e-test resourceVersion: "16216" uid: 7c1d153e-56ea-4e6f-9794-6883a5fad189 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-23504-predictor-557c78fdfd-f6js7 namespace: kserve-ci-e2e-test resourceVersion: "15745" uid: 7922d8ef-19c4-4ef9-a09b-f16db6cf33e5 kind: Event lastTimestamp: "2026-05-20T15:54:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T15:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:29Z" name: success-200-isvc-23504-predictor-557c78fdfd-f6js7.18b15112e68fa9ef namespace: kserve-ci-e2e-test resourceVersion: "20025" uid: 0b3a5d87-9200-475f-84e9-a15773261794 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-23504-predictor-557c78fdfd-f6js7 namespace: kserve-ci-e2e-test resourceVersion: "15745" uid: 7922d8ef-19c4-4ef9-a09b-f16db6cf33e5 kind: Event lastTimestamp: "2026-05-20T15:54:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:29Z" name: success-200-isvc-23504-predictor-557c78fdfd-f6js7.18b15112e6902c63 namespace: kserve-ci-e2e-test resourceVersion: "20026" uid: af83ffd8-4391-4a6a-bf22-d0ad2b2d0b8d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-23504-predictor-557c78fdfd namespace: kserve-ci-e2e-test resourceVersion: "15726" uid: dd0439ca-25c1-4db1-b148-909f0a894e45 kind: Event lastTimestamp: "2026-05-20T15:45:14Z" message: 'Created pod: success-200-isvc-23504-predictor-557c78fdfd-f6js7' metadata: creationTimestamp: "2026-05-20T15:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:14Z" name: success-200-isvc-23504-predictor-557c78fdfd.18b15091b6edf84e namespace: kserve-ci-e2e-test resourceVersion: "15746" uid: 84c3f75d-3005-4ab8-a9a4-0158f8bce75f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-23504-predictor namespace: kserve-ci-e2e-test resourceVersion: "15725" uid: b01dab41-b087-4d72-af2d-6bbd985a80e2 kind: Event lastTimestamp: "2026-05-20T15:45:14Z" message: Scaled up replica set success-200-isvc-23504-predictor-557c78fdfd from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:14Z" name: success-200-isvc-23504-predictor.18b15091b43fcdc1 namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: 164657a9-0c00-4550-be49-4b0407097cf0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:45:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-23504-predictor namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: 22f75b81-9a41-4b03-9f05-23316ee92842 kind: Event lastTimestamp: "2026-05-20T15:45:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:45:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:59Z" name: success-200-isvc-23504-predictor.18b150953367df87 namespace: kserve-ci-e2e-test resourceVersion: "16268" uid: 11417677-75be-44ec-a0ad-ea0d27ef649d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:45:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-23504-predictor namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: 22f75b81-9a41-4b03-9f05-23316ee92842 kind: Event lastTimestamp: "2026-05-20T15:45:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:45:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:59Z" name: success-200-isvc-23504-predictor.18b150953400eb8b namespace: kserve-ci-e2e-test resourceVersion: "16269" uid: 333c8a8c-45b6-4710-9f36-05d93a93c0e2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:46:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-23504-predictor namespace: kserve-ci-e2e-test resourceVersion: "15981" uid: 22f75b81-9a41-4b03-9f05-23316ee92842 kind: Event lastTimestamp: "2026-05-20T15:46:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:46:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:46:29Z" name: success-200-isvc-23504-predictor.18b1509fb0a206d3 namespace: kserve-ci-e2e-test resourceVersion: "16553" uid: 091fc23f-5c8e-4748-b341-f2f853cc9fef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:46:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-23504-predictor namespace: kserve-ci-e2e-test resourceVersion: "15981" uid: 22f75b81-9a41-4b03-9f05-23316ee92842 kind: Event lastTimestamp: "2026-05-20T15:46:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:46:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:46:29Z" name: success-200-isvc-23504-predictor.18b1509fb10b7bb1 namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: 256956cc-fdb3-43a1-9ffd-19fb0b772335 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T15:45:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-23504 namespace: kserve-ci-e2e-test resourceVersion: "15704" uid: 6e45891e-ad1b-4183-9aaf-d6f19deaa9d7 kind: Event lastTimestamp: "2026-05-20T15:46:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:46:03Z" name: success-200-isvc-23504.18b15091b4a0ffa2 namespace: kserve-ci-e2e-test resourceVersion: "16316" uid: 0d2ca6fd-5c9c-419a-8c14-d5c5ae56fc1f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-23504 namespace: kserve-ci-e2e-test resourceVersion: "16314" uid: 6e45891e-ad1b-4183-9aaf-d6f19deaa9d7 kind: Event lastTimestamp: "2026-05-20T15:46:03Z" message: InferenceService [success-200-isvc-23504] is Ready metadata: creationTimestamp: "2026-05-20T15:46:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:46:03Z" name: success-200-isvc-23504.18b1509d201bf3a8 namespace: kserve-ci-e2e-test resourceVersion: "16315" uid: 6da77846-d861-4125-bead-7b0a1722e098 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:55:23.970624Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c namespace: kserve-ci-e2e-test resourceVersion: "20688" uid: 022a7db3-b507-4127-9278-3d3308028d02 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T15:55:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:55:23Z" name: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c.18b1511f9ac71383 namespace: kserve-ci-e2e-test resourceVersion: "20694" uid: 7132ac5d-6169-4c9d-8253-0925f68d8d34 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:24Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: 022a7db3-b507-4127-9278-3d3308028d02 kind: Event lastTimestamp: "2026-05-20T15:55:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-3f928-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T15:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:24Z" name: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c.18b1511fa7f5e75c namespace: kserve-ci-e2e-test resourceVersion: "20743" uid: ad281a0f-3ee0-481f-8a73-9cbcd2889514 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c namespace: kserve-ci-e2e-test resourceVersion: "20708" uid: 022a7db3-b507-4127-9278-3d3308028d02 kind: Event lastTimestamp: "2026-05-20T15:55:25Z" message: Add eth0 [10.132.0.51/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:55:25Z" name: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c.18b1511fd924e1cd namespace: kserve-ci-e2e-test resourceVersion: "20755" uid: 5cf8aa80-9943-421c-9fa9-53875383e9bd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: 022a7db3-b507-4127-9278-3d3308028d02 kind: Event lastTimestamp: "2026-05-20T15:55:25Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-05-20T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:25Z" name: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c.18b1511fdc4251f0 namespace: kserve-ci-e2e-test resourceVersion: "20757" uid: 66878d4f-e058-4ebc-b91d-06938957a17f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: 022a7db3-b507-4127-9278-3d3308028d02 kind: Event lastTimestamp: "2026-05-20T15:55:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:25Z" name: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c.18b1511fe13de2a4 namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: 85ff4a3e-4068-4214-b5bc-b8d809e148cd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: 022a7db3-b507-4127-9278-3d3308028d02 kind: Event lastTimestamp: "2026-05-20T15:55:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:25Z" name: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c.18b1511fe1c9e791 namespace: kserve-ci-e2e-test resourceVersion: "20762" uid: 9d2423d8-845f-4a3a-a680-e7dc24fe1e6d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: 022a7db3-b507-4127-9278-3d3308028d02 kind: Event lastTimestamp: "2026-05-20T15:55:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:25Z" name: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c.18b1511fe1d73e7d namespace: kserve-ci-e2e-test resourceVersion: "20763" uid: 9f99c2d6-2172-487d-8a34-9f6d0eb1e720 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: 022a7db3-b507-4127-9278-3d3308028d02 kind: Event lastTimestamp: "2026-05-20T15:55:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:25Z" name: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c.18b1511fe4fa6938 namespace: kserve-ci-e2e-test resourceVersion: "20766" uid: ce837546-29e9-4f4b-8a2b-671874fc6935 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: 022a7db3-b507-4127-9278-3d3308028d02 kind: Event lastTimestamp: "2026-05-20T15:55:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:25Z" name: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c.18b1511fe57cc0a7 namespace: kserve-ci-e2e-test resourceVersion: "20769" uid: b6e6c86d-21e7-4167-a008-dbfcfdbf339c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-20T15:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: 022a7db3-b507-4127-9278-3d3308028d02 kind: Event lastTimestamp: "2026-05-20T16:04:51Z" message: 'Readiness probe failed: dial tcp 10.132.0.51:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:04:51Z" name: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c.18b15120016da2e9 namespace: kserve-ci-e2e-test resourceVersion: "25478" uid: 4948dc09-2acd-40c4-b5a7-f7ded254c044 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: 022a7db3-b507-4127-9278-3d3308028d02 kind: Event lastTimestamp: "2026-05-20T16:04:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T16:04:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:04:48Z" name: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c.18b151a32279480f namespace: kserve-ci-e2e-test resourceVersion: "25364" uid: fe007631-6de1-4ba0-8393-ebe64c314f57 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: 022a7db3-b507-4127-9278-3d3308028d02 kind: Event lastTimestamp: "2026-05-20T16:04:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T16:04:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:04:48Z" name: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c.18b151a32279d7ba namespace: kserve-ci-e2e-test resourceVersion: "25367" uid: 4917efbb-1e66-4265-a7b9-3baaf1a84b82 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: 022a7db3-b507-4127-9278-3d3308028d02 kind: Event lastTimestamp: "2026-05-20T16:04:51Z" message: 'Readiness probe failed: Get "https://10.132.0.51:8643/healthz": dial tcp 10.132.0.51:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T16:04:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:04:51Z" name: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c.18b151a3c9ca4a75 namespace: kserve-ci-e2e-test resourceVersion: "25476" uid: 7774c2e5-7779-4cc1-97f7-a09aaeb1c2cb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3f928-predictor-669ffb9b6b namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: 233ec103-efaa-4921-b1a4-8b1db19f8325 kind: Event lastTimestamp: "2026-05-20T15:55:23Z" message: 'Created pod: success-200-isvc-3f928-predictor-669ffb9b6b-6pg4c' metadata: creationTimestamp: "2026-05-20T15:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:55:23Z" name: success-200-isvc-3f928-predictor-669ffb9b6b.18b1511f9a2d8fd9 namespace: kserve-ci-e2e-test resourceVersion: "20691" uid: a64845c5-4a63-4d45-92df-2c4c9aa90665 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3f928-predictor namespace: kserve-ci-e2e-test resourceVersion: "20671" uid: 62078f6c-7291-4603-a626-2fbaf197db15 kind: Event lastTimestamp: "2026-05-20T15:55:23Z" message: Scaled up replica set success-200-isvc-3f928-predictor-669ffb9b6b from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:55:23Z" name: success-200-isvc-3f928-predictor.18b1511f9834ab03 namespace: kserve-ci-e2e-test resourceVersion: "20681" uid: e5c0e8de-5507-4667-af7d-1afb026956c9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:55:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3f928-predictor namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: 66032216-3276-4995-bc4d-3dffd8806bd7 kind: Event lastTimestamp: "2026-05-20T15:55:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:55:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:55:53Z" name: success-200-isvc-3f928-predictor.18b1512317ad2668 namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: 97faca34-8ac4-4802-a67b-8586493f2449 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:55:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3f928-predictor namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: 66032216-3276-4995-bc4d-3dffd8806bd7 kind: Event lastTimestamp: "2026-05-20T15:55:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:55:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:55:53Z" name: success-200-isvc-3f928-predictor.18b1512318255380 namespace: kserve-ci-e2e-test resourceVersion: "21222" uid: 11e95962-ac46-49a5-8caa-56104cf8cd40 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:56:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3f928-predictor namespace: kserve-ci-e2e-test resourceVersion: "20952" uid: 66032216-3276-4995-bc4d-3dffd8806bd7 kind: Event lastTimestamp: "2026-05-20T15:56:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:56:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:56:39Z" name: success-200-isvc-3f928-predictor.18b1512a17389541 namespace: kserve-ci-e2e-test resourceVersion: "21647" uid: 97b4c51f-28c7-4a8b-869d-8319e104c387 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:56:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3f928-predictor namespace: kserve-ci-e2e-test resourceVersion: "20952" uid: 66032216-3276-4995-bc4d-3dffd8806bd7 kind: Event lastTimestamp: "2026-05-20T15:56:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:56:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:56:39Z" name: success-200-isvc-3f928-predictor.18b1512a17b61a63 namespace: kserve-ci-e2e-test resourceVersion: "21648" uid: 12b9aaf0-d343-4a02-85d5-09accad5c8e1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T15:55:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3f928 namespace: kserve-ci-e2e-test resourceVersion: "20657" uid: 92bc7d18-df23-4f0c-b019-c68e65b6fb87 kind: Event lastTimestamp: "2026-05-20T15:56:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:56:21Z" name: success-200-isvc-3f928.18b1511f97e666e6 namespace: kserve-ci-e2e-test resourceVersion: "21460" uid: 6277b9aa-5e65-400e-805d-1e098b0f31aa reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3f928 namespace: kserve-ci-e2e-test resourceVersion: "20657" uid: 92bc7d18-df23-4f0c-b019-c68e65b6fb87 kind: Event lastTimestamp: "2026-05-20T15:55:23Z" message: 'Failed to update status for InferenceService "success-200-isvc-3f928": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3f928": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:55:23Z" name: success-200-isvc-3f928.18b1511f985442a5 namespace: kserve-ci-e2e-test resourceVersion: "20683" uid: a52339fb-c1e6-4cc0-bb59-5f1fbba331ab reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3f928 namespace: kserve-ci-e2e-test resourceVersion: "20657" uid: 92bc7d18-df23-4f0c-b019-c68e65b6fb87 kind: Event lastTimestamp: "2026-05-20T15:55:23Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3f928": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:55:23Z" name: success-200-isvc-3f928.18b1511f9854acec namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: ddac11f1-a603-459d-b095-5636c4a60eba reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:56:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3f928 namespace: kserve-ci-e2e-test resourceVersion: "21458" uid: 92bc7d18-df23-4f0c-b019-c68e65b6fb87 kind: Event lastTimestamp: "2026-05-20T15:56:21Z" message: InferenceService [success-200-isvc-3f928] is Ready metadata: creationTimestamp: "2026-05-20T15:56:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:56:21Z" name: success-200-isvc-3f928.18b1512d11106898 namespace: kserve-ci-e2e-test resourceVersion: "21459" uid: 3cff3cc7-546a-4340-ac7b-6c571443906c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:54:29.468989Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4ff76-predictor-5878d445d4-msxvp namespace: kserve-ci-e2e-test resourceVersion: "20043" uid: bbffac89-efb6-4741-b082-16c3d625f6b5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4ff76-predictor-5878d445d4-msxvp to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T15:54:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:54:29Z" name: success-200-isvc-4ff76-predictor-5878d445d4-msxvp.18b15112ea3b9f84 namespace: kserve-ci-e2e-test resourceVersion: "20052" uid: f646cf23-cc30-4c12-a9d2-db9a5ab5a27c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:29Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4ff76-predictor-5878d445d4-msxvp namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: bbffac89-efb6-4741-b082-16c3d625f6b5 kind: Event lastTimestamp: "2026-05-20T15:54:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-4ff76-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T15:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:29Z" name: success-200-isvc-4ff76-predictor-5878d445d4-msxvp.18b15112f816a746 namespace: kserve-ci-e2e-test resourceVersion: "20091" uid: dc9af39c-540a-4df6-b655-4df17ca7ae8b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4ff76-predictor-5878d445d4-msxvp namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: bbffac89-efb6-4741-b082-16c3d625f6b5 kind: Event lastTimestamp: "2026-05-20T15:54:30Z" message: Add eth0 [10.132.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:54:30Z" name: success-200-isvc-4ff76-predictor-5878d445d4-msxvp.18b151132ac0aad8 namespace: kserve-ci-e2e-test resourceVersion: "20115" uid: 7eec5ae9-35d0-44c8-92f9-b8a63d580d85 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4ff76-predictor-5878d445d4-msxvp namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: bbffac89-efb6-4741-b082-16c3d625f6b5 kind: Event lastTimestamp: "2026-05-20T15:54:30Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-05-20T15:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:30Z" name: success-200-isvc-4ff76-predictor-5878d445d4-msxvp.18b151132c5754f7 namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: ca5d9c59-b4f0-4a81-8eb9-06b97f891151 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4ff76-predictor-5878d445d4-msxvp namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: bbffac89-efb6-4741-b082-16c3d625f6b5 kind: Event lastTimestamp: "2026-05-20T15:54:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:30Z" name: success-200-isvc-4ff76-predictor-5878d445d4-msxvp.18b151132ff06e34 namespace: kserve-ci-e2e-test resourceVersion: "20120" uid: 635fbcc9-c9f5-4b82-80d2-07e5e645a669 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4ff76-predictor-5878d445d4-msxvp namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: bbffac89-efb6-4741-b082-16c3d625f6b5 kind: Event lastTimestamp: "2026-05-20T15:54:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:30Z" name: success-200-isvc-4ff76-predictor-5878d445d4-msxvp.18b15113307fb24e namespace: kserve-ci-e2e-test resourceVersion: "20121" uid: 07c50bed-d28b-4f70-b4c0-e50ea992eb07 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4ff76-predictor-5878d445d4-msxvp namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: bbffac89-efb6-4741-b082-16c3d625f6b5 kind: Event lastTimestamp: "2026-05-20T15:54:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T15:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:30Z" name: success-200-isvc-4ff76-predictor-5878d445d4-msxvp.18b15113308c9f5b namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: 9c41b25f-57cf-450b-8ceb-5434f999b5e3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4ff76-predictor-5878d445d4-msxvp namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: bbffac89-efb6-4741-b082-16c3d625f6b5 kind: Event lastTimestamp: "2026-05-20T15:54:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:30Z" name: success-200-isvc-4ff76-predictor-5878d445d4-msxvp.18b1511334065a2f namespace: kserve-ci-e2e-test resourceVersion: "20123" uid: 59268eaf-1554-479d-b0fb-f51edc631ead reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4ff76-predictor-5878d445d4-msxvp namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: bbffac89-efb6-4741-b082-16c3d625f6b5 kind: Event lastTimestamp: "2026-05-20T15:54:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:54:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:30Z" name: success-200-isvc-4ff76-predictor-5878d445d4-msxvp.18b1511334ac79f3 namespace: kserve-ci-e2e-test resourceVersion: "20124" uid: f096c6ed-8523-44c7-aa8a-00e422827a0c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T15:54:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4ff76-predictor-5878d445d4-msxvp namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: bbffac89-efb6-4741-b082-16c3d625f6b5 kind: Event lastTimestamp: "2026-05-20T15:55:08Z" message: 'Readiness probe failed: dial tcp 10.132.0.48:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:08Z" name: success-200-isvc-4ff76-predictor-5878d445d4-msxvp.18b151139e03803b namespace: kserve-ci-e2e-test resourceVersion: "20470" uid: 8a523080-4493-410d-aca0-03eb978a9384 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4ff76-predictor-5878d445d4-msxvp namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: bbffac89-efb6-4741-b082-16c3d625f6b5 kind: Event lastTimestamp: "2026-05-20T15:55:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T15:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:39Z" name: success-200-isvc-4ff76-predictor-5878d445d4-msxvp.18b151234a06c1ac namespace: kserve-ci-e2e-test resourceVersion: "21033" uid: 9edb5ca3-213e-48ad-b4ea-4e409a6c2801 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4ff76-predictor-5878d445d4-msxvp namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: bbffac89-efb6-4741-b082-16c3d625f6b5 kind: Event lastTimestamp: "2026-05-20T15:55:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:39Z" name: success-200-isvc-4ff76-predictor-5878d445d4-msxvp.18b151234a0740f9 namespace: kserve-ci-e2e-test resourceVersion: "21037" uid: a7ff5d40-0f7c-4acb-9a79-dcc8caba32d7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4ff76-predictor-5878d445d4-msxvp namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: bbffac89-efb6-4741-b082-16c3d625f6b5 kind: Event lastTimestamp: "2026-05-20T15:55:43Z" message: 'Readiness probe failed: Get "https://10.132.0.48:8643/healthz": dial tcp 10.132.0.48:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:55:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:43Z" name: success-200-isvc-4ff76-predictor-5878d445d4-msxvp.18b151242632ecac namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 7e85c787-deec-4c82-b249-8a31134668b1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4ff76-predictor-5878d445d4 namespace: kserve-ci-e2e-test resourceVersion: "20028" uid: 6cb62225-246a-4313-8a6e-0b2d5d2973d9 kind: Event lastTimestamp: "2026-05-20T15:54:29Z" message: 'Created pod: success-200-isvc-4ff76-predictor-5878d445d4-msxvp' metadata: creationTimestamp: "2026-05-20T15:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:54:29Z" name: success-200-isvc-4ff76-predictor-5878d445d4.18b15112e9dc2cb2 namespace: kserve-ci-e2e-test resourceVersion: "20049" uid: c7c26c5d-7e1d-4099-a9d7-23869c59064d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4ff76-predictor namespace: kserve-ci-e2e-test resourceVersion: "20024" uid: 70551f93-fc68-4dc0-a338-89af8767084d kind: Event lastTimestamp: "2026-05-20T15:54:29Z" message: Scaled up replica set success-200-isvc-4ff76-predictor-5878d445d4 from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:54:29Z" name: success-200-isvc-4ff76-predictor.18b15112e7c2acac namespace: kserve-ci-e2e-test resourceVersion: "20030" uid: 0001b5f1-fce1-4150-b985-8f15f32b6a12 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4ff76-predictor namespace: kserve-ci-e2e-test resourceVersion: "20027" uid: cfb6248d-f347-448c-a7d8-d1b4457b659b kind: Event lastTimestamp: "2026-05-20T15:54:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:54:44Z" name: success-200-isvc-4ff76-predictor.18b1511666e8ba37 namespace: kserve-ci-e2e-test resourceVersion: "20250" uid: bd92eb89-5741-4d01-aaa8-40b7331c08fe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4ff76-predictor namespace: kserve-ci-e2e-test resourceVersion: "20027" uid: cfb6248d-f347-448c-a7d8-d1b4457b659b kind: Event lastTimestamp: "2026-05-20T15:54:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:54:44Z" name: success-200-isvc-4ff76-predictor.18b15116675d3eef namespace: kserve-ci-e2e-test resourceVersion: "20252" uid: c3651d2b-2b94-4c5b-8ca7-4c0d5e228bbb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:54:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4ff76-predictor namespace: kserve-ci-e2e-test resourceVersion: "20251" uid: cfb6248d-f347-448c-a7d8-d1b4457b659b kind: Event lastTimestamp: "2026-05-20T15:55:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:55:29Z" name: success-200-isvc-4ff76-predictor.18b15119e626c1a7 namespace: kserve-ci-e2e-test resourceVersion: "20854" uid: c200405c-a519-4649-9baf-5ac170da7323 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:54:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4ff76-predictor namespace: kserve-ci-e2e-test resourceVersion: "20251" uid: cfb6248d-f347-448c-a7d8-d1b4457b659b kind: Event lastTimestamp: "2026-05-20T15:55:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:55:29Z" name: success-200-isvc-4ff76-predictor.18b15119e69ff24d namespace: kserve-ci-e2e-test resourceVersion: "20855" uid: 79a27387-2b3f-4658-950e-49ac57540619 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T15:54:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4ff76 namespace: kserve-ci-e2e-test resourceVersion: "20004" uid: a04dab2c-e7ed-4d15-a3b4-a58652a981eb kind: Event lastTimestamp: "2026-05-20T15:55:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:55:18Z" name: success-200-isvc-4ff76.18b15112e779904b namespace: kserve-ci-e2e-test resourceVersion: "20591" uid: 22d83736-f37a-4f1b-9b19-fbb55fa9c983 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4ff76 namespace: kserve-ci-e2e-test resourceVersion: "20004" uid: a04dab2c-e7ed-4d15-a3b4-a58652a981eb kind: Event lastTimestamp: "2026-05-20T15:54:29Z" message: 'Failed to update status for InferenceService "success-200-isvc-4ff76": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4ff76": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:54:29Z" name: success-200-isvc-4ff76.18b15112e8005943 namespace: kserve-ci-e2e-test resourceVersion: "20032" uid: 8536e3be-400c-445b-8857-e55b61a12445 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4ff76 namespace: kserve-ci-e2e-test resourceVersion: "20004" uid: a04dab2c-e7ed-4d15-a3b4-a58652a981eb kind: Event lastTimestamp: "2026-05-20T15:54:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4ff76": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:54:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:54:29Z" name: success-200-isvc-4ff76.18b15112e800adb4 namespace: kserve-ci-e2e-test resourceVersion: "20034" uid: 1bf67780-286a-4757-a030-500020cc971e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4ff76 namespace: kserve-ci-e2e-test resourceVersion: "20589" uid: a04dab2c-e7ed-4d15-a3b4-a58652a981eb kind: Event lastTimestamp: "2026-05-20T15:55:18Z" message: InferenceService [success-200-isvc-4ff76] is Ready metadata: creationTimestamp: "2026-05-20T15:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:55:18Z" name: success-200-isvc-4ff76.18b1511e59c860ef namespace: kserve-ci-e2e-test resourceVersion: "20590" uid: 0f0c1035-2040-49f1-b935-93b5ce9711ef reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:43:04.208636Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-542af-predictor-6f849b64fd-lbxwr namespace: kserve-ci-e2e-test resourceVersion: "14115" uid: 5f54aa33-a2d5-4ac1-8ff3-d4bc9b747ed3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-542af-predictor-6f849b64fd-lbxwr to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T15:43:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:43:04Z" name: success-200-isvc-542af-predictor-6f849b64fd-lbxwr.18b150735d870c02 namespace: kserve-ci-e2e-test resourceVersion: "14120" uid: ceede1b7-ac9f-4dee-ab2b-6417475fdb44 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:04Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-542af-predictor-6f849b64fd-lbxwr namespace: kserve-ci-e2e-test resourceVersion: "14127" uid: 5f54aa33-a2d5-4ac1-8ff3-d4bc9b747ed3 kind: Event lastTimestamp: "2026-05-20T15:43:04Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:43:04Z" name: success-200-isvc-542af-predictor-6f849b64fd-lbxwr.18b1507379c4d09d namespace: kserve-ci-e2e-test resourceVersion: "14202" uid: af1820da-d937-4c37-aab5-c5851c39d3c5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-542af-predictor-6f849b64fd-lbxwr namespace: kserve-ci-e2e-test resourceVersion: "14117" uid: 5f54aa33-a2d5-4ac1-8ff3-d4bc9b747ed3 kind: Event lastTimestamp: "2026-05-20T15:43:04Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1433" metadata: creationTimestamp: "2026-05-20T15:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:04Z" name: success-200-isvc-542af-predictor-6f849b64fd-lbxwr.18b150737b3ed295 namespace: kserve-ci-e2e-test resourceVersion: "14206" uid: b8d6aaa3-b920-406b-82f3-951821c0ab50 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-542af-predictor-6f849b64fd-lbxwr namespace: kserve-ci-e2e-test resourceVersion: "14117" uid: 5f54aa33-a2d5-4ac1-8ff3-d4bc9b747ed3 kind: Event lastTimestamp: "2026-05-20T15:43:20Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1433" in 15.493s (15.493s including waiting). Image size: 1335714912 bytes.' metadata: creationTimestamp: "2026-05-20T15:43:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:20Z" name: success-200-isvc-542af-predictor-6f849b64fd-lbxwr.18b1507716bcc7b2 namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: f41ba566-cffc-4f1c-aded-f0f8d0eabfcb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-542af-predictor-6f849b64fd-lbxwr namespace: kserve-ci-e2e-test resourceVersion: "14117" uid: 5f54aa33-a2d5-4ac1-8ff3-d4bc9b747ed3 kind: Event lastTimestamp: "2026-05-20T15:43:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:43:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:20Z" name: success-200-isvc-542af-predictor-6f849b64fd-lbxwr.18b150771ceea1ed namespace: kserve-ci-e2e-test resourceVersion: "14439" uid: 8f6ff15b-3bf9-4371-8b85-42d0e754d77a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-542af-predictor-6f849b64fd-lbxwr namespace: kserve-ci-e2e-test resourceVersion: "14117" uid: 5f54aa33-a2d5-4ac1-8ff3-d4bc9b747ed3 kind: Event lastTimestamp: "2026-05-20T15:43:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:43:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:20Z" name: success-200-isvc-542af-predictor-6f849b64fd-lbxwr.18b150771de5193f namespace: kserve-ci-e2e-test resourceVersion: "14441" uid: efb415d6-658c-4bf5-ae43-747328d0f958 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-542af-predictor-6f849b64fd-lbxwr namespace: kserve-ci-e2e-test resourceVersion: "14117" uid: 5f54aa33-a2d5-4ac1-8ff3-d4bc9b747ed3 kind: Event lastTimestamp: "2026-05-20T15:43:20Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-20T15:43:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:20Z" name: success-200-isvc-542af-predictor-6f849b64fd-lbxwr.18b150771df2d55e namespace: kserve-ci-e2e-test resourceVersion: "14442" uid: eef8d8a9-7546-4c8d-8d85-360b19d12f56 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-542af-predictor-6f849b64fd-lbxwr namespace: kserve-ci-e2e-test resourceVersion: "14117" uid: 5f54aa33-a2d5-4ac1-8ff3-d4bc9b747ed3 kind: Event lastTimestamp: "2026-05-20T15:43:24Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.632s (4.632s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-20T15:43:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:24Z" name: success-200-isvc-542af-predictor-6f849b64fd-lbxwr.18b15078320cd59f namespace: kserve-ci-e2e-test resourceVersion: "14481" uid: 0959b4e2-6eb2-422e-8c15-5ce9ed3f1580 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-542af-predictor-6f849b64fd-lbxwr namespace: kserve-ci-e2e-test resourceVersion: "14117" uid: 5f54aa33-a2d5-4ac1-8ff3-d4bc9b747ed3 kind: Event lastTimestamp: "2026-05-20T15:43:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:25Z" name: success-200-isvc-542af-predictor-6f849b64fd-lbxwr.18b1507836a5d7fc namespace: kserve-ci-e2e-test resourceVersion: "14483" uid: 8f31b5a1-baf3-42e3-9fa2-a37081dc6efb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-542af-predictor-6f849b64fd-lbxwr namespace: kserve-ci-e2e-test resourceVersion: "14117" uid: 5f54aa33-a2d5-4ac1-8ff3-d4bc9b747ed3 kind: Event lastTimestamp: "2026-05-20T15:43:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:43:25Z" name: success-200-isvc-542af-predictor-6f849b64fd-lbxwr.18b150783732c049 namespace: kserve-ci-e2e-test resourceVersion: "14485" uid: 705b78a3-416a-4096-826e-b7f2eceed7e8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T15:43:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-542af-predictor-6f849b64fd-lbxwr namespace: kserve-ci-e2e-test resourceVersion: "14117" uid: 5f54aa33-a2d5-4ac1-8ff3-d4bc9b747ed3 kind: Event lastTimestamp: "2026-05-20T15:44:03Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:43:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:03Z" name: success-200-isvc-542af-predictor-6f849b64fd-lbxwr.18b15078aeafe537 namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: 801f538b-13cb-4920-8d74-8f4b922df1f6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-542af-predictor-6f849b64fd-lbxwr namespace: kserve-ci-e2e-test resourceVersion: "14117" uid: 5f54aa33-a2d5-4ac1-8ff3-d4bc9b747ed3 kind: Event lastTimestamp: "2026-05-20T15:44:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T15:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:38Z" name: success-200-isvc-542af-predictor-6f849b64fd-lbxwr.18b150895ee35ed2 namespace: kserve-ci-e2e-test resourceVersion: "15239" uid: beab58a8-9d34-4109-a02b-b42223f36594 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-542af-predictor-6f849b64fd-lbxwr namespace: kserve-ci-e2e-test resourceVersion: "14117" uid: 5f54aa33-a2d5-4ac1-8ff3-d4bc9b747ed3 kind: Event lastTimestamp: "2026-05-20T15:44:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:38Z" name: success-200-isvc-542af-predictor-6f849b64fd-lbxwr.18b150895ee3f484 namespace: kserve-ci-e2e-test resourceVersion: "15242" uid: 76d88cdc-164c-4f03-8fcb-63965591a380 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-542af-predictor-6f849b64fd namespace: kserve-ci-e2e-test resourceVersion: "14107" uid: 027e4d9c-d957-4283-8803-c749566ef6bd kind: Event lastTimestamp: "2026-05-20T15:43:04Z" message: 'Created pod: success-200-isvc-542af-predictor-6f849b64fd-lbxwr' metadata: creationTimestamp: "2026-05-20T15:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:43:04Z" name: success-200-isvc-542af-predictor-6f849b64fd.18b150735d11696e namespace: kserve-ci-e2e-test resourceVersion: "14116" uid: 710a4859-e68c-43b6-984f-40bd97a1af45 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-542af-predictor namespace: kserve-ci-e2e-test resourceVersion: "14105" uid: 0f9183b8-7969-47e9-9d5d-5271a7c8fc0a kind: Event lastTimestamp: "2026-05-20T15:43:04Z" message: Scaled up replica set success-200-isvc-542af-predictor-6f849b64fd from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:43:04Z" name: success-200-isvc-542af-predictor.18b1507357cd178b namespace: kserve-ci-e2e-test resourceVersion: "14108" uid: 97fdad44-9088-4dd8-9648-0e4f0304b763 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:43:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-542af-predictor namespace: kserve-ci-e2e-test resourceVersion: "14119" uid: 541b03d2-2699-4de2-9624-3f602b6666a9 kind: Event lastTimestamp: "2026-05-20T15:44:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:04Z" name: success-200-isvc-542af-predictor.18b15076de2770ed namespace: kserve-ci-e2e-test resourceVersion: "14840" uid: 90619292-494b-4bce-93dc-42f36f75c290 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:43:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-542af-predictor namespace: kserve-ci-e2e-test resourceVersion: "14119" uid: 541b03d2-2699-4de2-9624-3f602b6666a9 kind: Event lastTimestamp: "2026-05-20T15:44:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:04Z" name: success-200-isvc-542af-predictor.18b15076deb85d82 namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 25029236-b0d5-4416-8757-99cf587cf4c8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:44:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-542af-predictor namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: 541b03d2-2699-4de2-9624-3f602b6666a9 kind: Event lastTimestamp: "2026-05-20T15:44:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:34Z" name: success-200-isvc-542af-predictor.18b15084ddb977f3 namespace: kserve-ci-e2e-test resourceVersion: "15139" uid: ac311167-9f34-4bd2-ae03-afeaebb4a2b2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:44:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-542af-predictor namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: 541b03d2-2699-4de2-9624-3f602b6666a9 kind: Event lastTimestamp: "2026-05-20T15:44:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:34Z" name: success-200-isvc-542af-predictor.18b15084de46263f namespace: kserve-ci-e2e-test resourceVersion: "15140" uid: 18772c9e-f78c-4d8b-a2b8-6ac473ce1940 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T15:43:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-542af namespace: kserve-ci-e2e-test resourceVersion: "14095" uid: 9cb7d101-91ba-4dfe-a621-0d4486f37137 kind: Event lastTimestamp: "2026-05-20T15:44:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:44:13Z" name: success-200-isvc-542af.18b150735dbf3712 namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: 174fcb33-1cfa-4020-918e-c055acb5cc0d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-542af namespace: kserve-ci-e2e-test resourceVersion: "14095" uid: 9cb7d101-91ba-4dfe-a621-0d4486f37137 kind: Event lastTimestamp: "2026-05-20T15:43:04Z" message: 'Failed to update status for InferenceService "success-200-isvc-542af": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-542af": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:43:04Z" name: success-200-isvc-542af.18b150735eb00799 namespace: kserve-ci-e2e-test resourceVersion: "14124" uid: 30fc4552-a08b-4fc4-82c9-a8d56f527c4a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:43:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-542af namespace: kserve-ci-e2e-test resourceVersion: "14095" uid: 9cb7d101-91ba-4dfe-a621-0d4486f37137 kind: Event lastTimestamp: "2026-05-20T15:43:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-542af": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:43:04Z" name: success-200-isvc-542af.18b150735eb083fb namespace: kserve-ci-e2e-test resourceVersion: "14126" uid: 2298fe9e-0cc6-4860-b830-3fc727db7df8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-542af namespace: kserve-ci-e2e-test resourceVersion: "14922" uid: 9cb7d101-91ba-4dfe-a621-0d4486f37137 kind: Event lastTimestamp: "2026-05-20T15:44:13Z" message: InferenceService [success-200-isvc-542af] is Ready metadata: creationTimestamp: "2026-05-20T15:44:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:44:13Z" name: success-200-isvc-542af.18b150836cac1377 namespace: kserve-ci-e2e-test resourceVersion: "14923" uid: f69e8b3a-857a-4535-8904-b1e61b918246 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T16:06:09.296615Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-62c44-predictor-7c56bb66c6-6jdmp namespace: kserve-ci-e2e-test resourceVersion: "26391" uid: ffc0ed77-16ae-45e9-b1e7-19f1c4b44fed kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-62c44-predictor-7c56bb66c6-6jdmp to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T16:06:09Z" name: success-200-isvc-62c44-predictor-7c56bb66c6-6jdmp.18b151b5db345a75 namespace: kserve-ci-e2e-test resourceVersion: "26397" uid: 9b8c7e5a-bf39-4384-9ea3-a1389c64d7cf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-62c44-predictor-7c56bb66c6-6jdmp namespace: kserve-ci-e2e-test resourceVersion: "26412" uid: ffc0ed77-16ae-45e9-b1e7-19f1c4b44fed kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: Add eth0 [10.132.0.61/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T16:06:09Z" name: success-200-isvc-62c44-predictor-7c56bb66c6-6jdmp.18b151b5f5bd5abb namespace: kserve-ci-e2e-test resourceVersion: "26451" uid: f4f5dd07-13d0-40f3-af1a-1647d3c3928b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-62c44-predictor-7c56bb66c6-6jdmp namespace: kserve-ci-e2e-test resourceVersion: "26395" uid: ffc0ed77-16ae-45e9-b1e7-19f1c4b44fed kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:09Z" name: success-200-isvc-62c44-predictor-7c56bb66c6-6jdmp.18b151b5f71f2ace namespace: kserve-ci-e2e-test resourceVersion: "26453" uid: 545153ce-6ce9-4c02-8485-7f55ecf28a02 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-62c44-predictor-7c56bb66c6-6jdmp namespace: kserve-ci-e2e-test resourceVersion: "26395" uid: ffc0ed77-16ae-45e9-b1e7-19f1c4b44fed kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:09Z" name: success-200-isvc-62c44-predictor-7c56bb66c6-6jdmp.18b151b5fa841a40 namespace: kserve-ci-e2e-test resourceVersion: "26454" uid: e320bd3f-fe1d-4e52-b4ec-d8f39ce44cde reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-62c44-predictor-7c56bb66c6-6jdmp namespace: kserve-ci-e2e-test resourceVersion: "26395" uid: ffc0ed77-16ae-45e9-b1e7-19f1c4b44fed kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:09Z" name: success-200-isvc-62c44-predictor-7c56bb66c6-6jdmp.18b151b5fb06570e namespace: kserve-ci-e2e-test resourceVersion: "26455" uid: 40d292cb-db75-4368-a204-96cd5894f4f6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-62c44-predictor-7c56bb66c6-6jdmp namespace: kserve-ci-e2e-test resourceVersion: "26395" uid: ffc0ed77-16ae-45e9-b1e7-19f1c4b44fed kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:09Z" name: success-200-isvc-62c44-predictor-7c56bb66c6-6jdmp.18b151b5fb1f7ba5 namespace: kserve-ci-e2e-test resourceVersion: "26456" uid: c04cfaaf-ee40-40dc-b252-b9dd77e57b71 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-62c44-predictor-7c56bb66c6-6jdmp namespace: kserve-ci-e2e-test resourceVersion: "26395" uid: ffc0ed77-16ae-45e9-b1e7-19f1c4b44fed kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:09Z" name: success-200-isvc-62c44-predictor-7c56bb66c6-6jdmp.18b151b5fec691b6 namespace: kserve-ci-e2e-test resourceVersion: "26460" uid: d214b2ae-8685-4a08-a3b3-8325c33a6360 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-62c44-predictor-7c56bb66c6-6jdmp namespace: kserve-ci-e2e-test resourceVersion: "26395" uid: ffc0ed77-16ae-45e9-b1e7-19f1c4b44fed kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:09Z" name: success-200-isvc-62c44-predictor-7c56bb66c6-6jdmp.18b151b5ff49c8d0 namespace: kserve-ci-e2e-test resourceVersion: "26461" uid: 35603281-3885-445e-8672-2d6d23848368 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-20T16:06:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-62c44-predictor-7c56bb66c6-6jdmp namespace: kserve-ci-e2e-test resourceVersion: "26395" uid: ffc0ed77-16ae-45e9-b1e7-19f1c4b44fed kind: Event lastTimestamp: "2026-05-20T16:15:26Z" message: 'Readiness probe failed: dial tcp 10.132.0.61:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T16:06:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:15:26Z" name: success-200-isvc-62c44-predictor-7c56bb66c6-6jdmp.18b151b605a028cb namespace: kserve-ci-e2e-test resourceVersion: "30513" uid: 8f8b6f34-5264-4161-9812-65f17d7e3781 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:15:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-62c44-predictor-7c56bb66c6-6jdmp namespace: kserve-ci-e2e-test resourceVersion: "26395" uid: ffc0ed77-16ae-45e9-b1e7-19f1c4b44fed kind: Event lastTimestamp: "2026-05-20T16:15:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T16:15:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:15:24Z" name: success-200-isvc-62c44-predictor-7c56bb66c6-6jdmp.18b152370e895dc4 namespace: kserve-ci-e2e-test resourceVersion: "30484" uid: 50bc6590-98d9-43c4-806e-a4d2a9789f91 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:15:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-62c44-predictor-7c56bb66c6-6jdmp namespace: kserve-ci-e2e-test resourceVersion: "26395" uid: ffc0ed77-16ae-45e9-b1e7-19f1c4b44fed kind: Event lastTimestamp: "2026-05-20T16:15:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T16:15:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:15:24Z" name: success-200-isvc-62c44-predictor-7c56bb66c6-6jdmp.18b152370e89d483 namespace: kserve-ci-e2e-test resourceVersion: "30486" uid: 4c9f08ef-5a54-47e3-95ef-029c2a021673 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:15:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-62c44-predictor-7c56bb66c6-6jdmp namespace: kserve-ci-e2e-test resourceVersion: "26395" uid: ffc0ed77-16ae-45e9-b1e7-19f1c4b44fed kind: Event lastTimestamp: "2026-05-20T16:15:26Z" message: 'Readiness probe failed: Get "https://10.132.0.61:8643/healthz": dial tcp 10.132.0.61:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T16:15:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:15:26Z" name: success-200-isvc-62c44-predictor-7c56bb66c6-6jdmp.18b152377a11e694 namespace: kserve-ci-e2e-test resourceVersion: "30512" uid: a5cb55a3-b26e-48dc-8701-24508f4a34fd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-62c44-predictor-7c56bb66c6 namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: 91252558-f121-4413-8ab4-ac5295305399 kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: 'Created pod: success-200-isvc-62c44-predictor-7c56bb66c6-6jdmp' metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:06:09Z" name: success-200-isvc-62c44-predictor-7c56bb66c6.18b151b5dad6f77e namespace: kserve-ci-e2e-test resourceVersion: "26394" uid: eae9a053-6047-43ee-ae99-5a3d99cdb0c3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-62c44-predictor namespace: kserve-ci-e2e-test resourceVersion: "26375" uid: dcab624c-9891-4bc9-af66-fea61abf1431 kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: Scaled up replica set success-200-isvc-62c44-predictor-7c56bb66c6 from 0 to 1 metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:06:09Z" name: success-200-isvc-62c44-predictor.18b151b5d907fcbc namespace: kserve-ci-e2e-test resourceVersion: "26383" uid: 1449f1f0-6d44-4515-a97c-b3ca4a3cbfef reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T16:06:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-62c44-predictor namespace: kserve-ci-e2e-test resourceVersion: "26379" uid: 0ea19523-39ce-4c2c-8e2f-24202ac2f903 kind: Event lastTimestamp: "2026-05-20T16:06:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T16:06:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:06:39Z" name: success-200-isvc-62c44-predictor.18b151b9581fe5b8 namespace: kserve-ci-e2e-test resourceVersion: "26764" uid: f300ca16-2e1b-4dfb-ab70-7b05bd2003d4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T16:06:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-62c44-predictor namespace: kserve-ci-e2e-test resourceVersion: "26379" uid: 0ea19523-39ce-4c2c-8e2f-24202ac2f903 kind: Event lastTimestamp: "2026-05-20T16:06:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T16:06:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:06:39Z" name: success-200-isvc-62c44-predictor.18b151b958bbc916 namespace: kserve-ci-e2e-test resourceVersion: "26765" uid: f1abe410-6e41-4c44-bb94-6594c3bed151 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T16:06:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-62c44-predictor namespace: kserve-ci-e2e-test resourceVersion: "26643" uid: 0ea19523-39ce-4c2c-8e2f-24202ac2f903 kind: Event lastTimestamp: "2026-05-20T16:07:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T16:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:07:24Z" name: success-200-isvc-62c44-predictor.18b151c056f0a869 namespace: kserve-ci-e2e-test resourceVersion: "27153" uid: e8bf9943-5a89-423d-a8f9-dc0ea12e5d68 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T16:06:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-62c44-predictor namespace: kserve-ci-e2e-test resourceVersion: "26643" uid: 0ea19523-39ce-4c2c-8e2f-24202ac2f903 kind: Event lastTimestamp: "2026-05-20T16:07:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T16:06:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:07:24Z" name: success-200-isvc-62c44-predictor.18b151c0576f575d namespace: kserve-ci-e2e-test resourceVersion: "27154" uid: e5154113-89f5-42a2-8e0c-eb6351a9798a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-62c44 namespace: kserve-ci-e2e-test resourceVersion: "26359" uid: 852d1db5-434d-41be-b5e3-8e94c9e3c8fd kind: Event lastTimestamp: "2026-05-20T16:06:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:06:56Z" name: success-200-isvc-62c44.18b151b5d8d36493 namespace: kserve-ci-e2e-test resourceVersion: "26914" uid: 3da62091-d4bf-4579-8991-ac087760d373 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-62c44 namespace: kserve-ci-e2e-test resourceVersion: "26359" uid: 852d1db5-434d-41be-b5e3-8e94c9e3c8fd kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: 'Failed to update status for InferenceService "success-200-isvc-62c44": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-62c44": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:06:09Z" name: success-200-isvc-62c44.18b151b5d96cb183 namespace: kserve-ci-e2e-test resourceVersion: "26385" uid: 00460c78-b425-44fc-83f7-58bec262bb0a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-62c44 namespace: kserve-ci-e2e-test resourceVersion: "26359" uid: 852d1db5-434d-41be-b5e3-8e94c9e3c8fd kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-62c44": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:06:09Z" name: success-200-isvc-62c44.18b151b5d96d20a4 namespace: kserve-ci-e2e-test resourceVersion: "26387" uid: b351566a-3576-48c2-b44b-dd5782906fbb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-62c44 namespace: kserve-ci-e2e-test resourceVersion: "26910" uid: 852d1db5-434d-41be-b5e3-8e94c9e3c8fd kind: Event lastTimestamp: "2026-05-20T16:06:56Z" message: InferenceService [success-200-isvc-62c44] is Ready metadata: creationTimestamp: "2026-05-20T16:06:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:06:56Z" name: success-200-isvc-62c44.18b151c0c52bbe8c namespace: kserve-ci-e2e-test resourceVersion: "26911" uid: 9d6a6eab-1f9b-4062-a1e3-d3fb450a8f75 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:54:03.763741Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-687f8-predictor-7d75c44d4d-64c45 namespace: kserve-ci-e2e-test resourceVersion: "19714" uid: 41bf512d-9315-457b-a451-d7fb1eb230d4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-687f8-predictor-7d75c44d4d-64c45 to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T15:54:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:54:03Z" name: success-200-isvc-687f8-predictor-7d75c44d4d-64c45.18b1510cee13392f namespace: kserve-ci-e2e-test resourceVersion: "19725" uid: 40623c21-3d78-4b89-ab71-422d59c7003f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:03Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-687f8-predictor-7d75c44d4d-64c45 namespace: kserve-ci-e2e-test resourceVersion: "19718" uid: 41bf512d-9315-457b-a451-d7fb1eb230d4 kind: Event lastTimestamp: "2026-05-20T15:54:03Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-687f8-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T15:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:03Z" name: success-200-isvc-687f8-predictor-7d75c44d4d-64c45.18b1510cfad41150 namespace: kserve-ci-e2e-test resourceVersion: "19753" uid: 1e4b8b54-433c-4078-a3e2-453529d466a1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:04Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-687f8-predictor-7d75c44d4d-64c45 namespace: kserve-ci-e2e-test resourceVersion: "19735" uid: 41bf512d-9315-457b-a451-d7fb1eb230d4 kind: Event lastTimestamp: "2026-05-20T15:54:04Z" message: Add eth0 [10.132.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:54:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:54:04Z" name: success-200-isvc-687f8-predictor-7d75c44d4d-64c45.18b1510d2ca5db1e namespace: kserve-ci-e2e-test resourceVersion: "19771" uid: 4c40cc0f-0e6a-4097-a807-9ec43383a8c8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-687f8-predictor-7d75c44d4d-64c45 namespace: kserve-ci-e2e-test resourceVersion: "19718" uid: 41bf512d-9315-457b-a451-d7fb1eb230d4 kind: Event lastTimestamp: "2026-05-20T15:54:04Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-05-20T15:54:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:04Z" name: success-200-isvc-687f8-predictor-7d75c44d4d-64c45.18b1510d2e070d08 namespace: kserve-ci-e2e-test resourceVersion: "19773" uid: 79ce83ab-2142-4e27-a678-2bdbdc8f69e0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-687f8-predictor-7d75c44d4d-64c45 namespace: kserve-ci-e2e-test resourceVersion: "19718" uid: 41bf512d-9315-457b-a451-d7fb1eb230d4 kind: Event lastTimestamp: "2026-05-20T15:54:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:54:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:04Z" name: success-200-isvc-687f8-predictor-7d75c44d4d-64c45.18b1510d31b978e1 namespace: kserve-ci-e2e-test resourceVersion: "19775" uid: 424a29bb-ccf2-41f3-a8b7-1e01814a5d49 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-687f8-predictor-7d75c44d4d-64c45 namespace: kserve-ci-e2e-test resourceVersion: "19718" uid: 41bf512d-9315-457b-a451-d7fb1eb230d4 kind: Event lastTimestamp: "2026-05-20T15:54:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:54:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:04Z" name: success-200-isvc-687f8-predictor-7d75c44d4d-64c45.18b1510d32334c37 namespace: kserve-ci-e2e-test resourceVersion: "19776" uid: 6d2acfe8-3f67-4014-87c7-0edceea17311 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-687f8-predictor-7d75c44d4d-64c45 namespace: kserve-ci-e2e-test resourceVersion: "19718" uid: 41bf512d-9315-457b-a451-d7fb1eb230d4 kind: Event lastTimestamp: "2026-05-20T15:54:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T15:54:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:04Z" name: success-200-isvc-687f8-predictor-7d75c44d4d-64c45.18b1510d323ff945 namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: 1da2793a-9f4e-447f-92c6-c4b22a6fb7d6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-687f8-predictor-7d75c44d4d-64c45 namespace: kserve-ci-e2e-test resourceVersion: "19718" uid: 41bf512d-9315-457b-a451-d7fb1eb230d4 kind: Event lastTimestamp: "2026-05-20T15:54:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:54:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:04Z" name: success-200-isvc-687f8-predictor-7d75c44d4d-64c45.18b1510d34ee6216 namespace: kserve-ci-e2e-test resourceVersion: "19778" uid: 05728f2d-2cee-4168-997c-a02b95093bcb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-687f8-predictor-7d75c44d4d-64c45 namespace: kserve-ci-e2e-test resourceVersion: "19718" uid: 41bf512d-9315-457b-a451-d7fb1eb230d4 kind: Event lastTimestamp: "2026-05-20T15:54:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:54:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:04Z" name: success-200-isvc-687f8-predictor-7d75c44d4d-64c45.18b1510d3563f1ef namespace: kserve-ci-e2e-test resourceVersion: "19779" uid: 60bfba16-68c5-4908-a1f1-72c573ef0485 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-20T15:54:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-687f8-predictor-7d75c44d4d-64c45 namespace: kserve-ci-e2e-test resourceVersion: "19718" uid: 41bf512d-9315-457b-a451-d7fb1eb230d4 kind: Event lastTimestamp: "2026-05-20T15:54:51Z" message: 'Readiness probe failed: dial tcp 10.132.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:54:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:51Z" name: success-200-isvc-687f8-predictor-7d75c44d4d-64c45.18b1510d4dcf0285 namespace: kserve-ci-e2e-test resourceVersion: "20314" uid: a7321377-6230-4a11-8a8d-6a2953786639 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-687f8-predictor-7d75c44d4d-64c45 namespace: kserve-ci-e2e-test resourceVersion: "19718" uid: 41bf512d-9315-457b-a451-d7fb1eb230d4 kind: Event lastTimestamp: "2026-05-20T15:55:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T15:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:23Z" name: success-200-isvc-687f8-predictor-7d75c44d4d-64c45.18b1511f98ac0110 namespace: kserve-ci-e2e-test resourceVersion: "20684" uid: 83798502-bee2-41d5-a95b-649cab133beb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-687f8-predictor-7d75c44d4d-64c45 namespace: kserve-ci-e2e-test resourceVersion: "19718" uid: 41bf512d-9315-457b-a451-d7fb1eb230d4 kind: Event lastTimestamp: "2026-05-20T15:55:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:23Z" name: success-200-isvc-687f8-predictor-7d75c44d4d-64c45.18b1511f98ae5071 namespace: kserve-ci-e2e-test resourceVersion: "20686" uid: 23b568b7-6102-471a-afdb-3eae7b04c8b0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-687f8-predictor-7d75c44d4d-64c45 namespace: kserve-ci-e2e-test resourceVersion: "19718" uid: 41bf512d-9315-457b-a451-d7fb1eb230d4 kind: Event lastTimestamp: "2026-05-20T15:55:26Z" message: 'Readiness probe failed: Get "https://10.132.0.46:8643/healthz": dial tcp 10.132.0.46:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:26Z" name: success-200-isvc-687f8-predictor-7d75c44d4d-64c45.18b1512029ffcc25 namespace: kserve-ci-e2e-test resourceVersion: "20791" uid: bff9312a-a110-42b5-bbff-407ac94a1882 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-687f8-predictor-7d75c44d4d namespace: kserve-ci-e2e-test resourceVersion: "19689" uid: 8ed897ed-f4fc-4d5e-81e6-5769031f6568 kind: Event lastTimestamp: "2026-05-20T15:54:03Z" message: 'Created pod: success-200-isvc-687f8-predictor-7d75c44d4d-64c45' metadata: creationTimestamp: "2026-05-20T15:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:54:03Z" name: success-200-isvc-687f8-predictor-7d75c44d4d.18b1510cedbf29c5 namespace: kserve-ci-e2e-test resourceVersion: "19720" uid: 9549c83a-fa7e-4284-b31d-37fc1c58752b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-687f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19686" uid: 6a7e2072-99af-4736-bfbb-b05a0083c5a2 kind: Event lastTimestamp: "2026-05-20T15:54:03Z" message: Scaled up replica set success-200-isvc-687f8-predictor-7d75c44d4d from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:54:03Z" name: success-200-isvc-687f8-predictor.18b1510ce6760759 namespace: kserve-ci-e2e-test resourceVersion: "19693" uid: 587d4c6d-e495-4ca0-bc95-aba087a4d135 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:54:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-687f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19688" uid: cfe9e400-2342-49c2-8922-daf765327877 kind: Event lastTimestamp: "2026-05-20T15:54:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:54:33Z" name: success-200-isvc-687f8-predictor.18b1511065177215 namespace: kserve-ci-e2e-test resourceVersion: "20157" uid: a5259fa2-4e1d-4485-acef-ec5a3e078e3d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:54:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-687f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19688" uid: cfe9e400-2342-49c2-8922-daf765327877 kind: Event lastTimestamp: "2026-05-20T15:54:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:54:33Z" name: success-200-isvc-687f8-predictor.18b151106590c14f namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: 56653706-4a88-41f6-9fa5-3c72f23c4e6a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:54:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-687f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19897" uid: cfe9e400-2342-49c2-8922-daf765327877 kind: Event lastTimestamp: "2026-05-20T15:55:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:55:18Z" name: success-200-isvc-687f8-predictor.18b151176341cf5b namespace: kserve-ci-e2e-test resourceVersion: "20592" uid: f4bc25d8-4771-48cd-9407-16a0dcf9edb9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:54:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-687f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19897" uid: cfe9e400-2342-49c2-8922-daf765327877 kind: Event lastTimestamp: "2026-05-20T15:55:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:55:18Z" name: success-200-isvc-687f8-predictor.18b1511763bd1634 namespace: kserve-ci-e2e-test resourceVersion: "20593" uid: 269c2625-b19f-41fc-9bff-2b6b03bd68a6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T15:54:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-687f8 namespace: kserve-ci-e2e-test resourceVersion: "19666" uid: 7c694c0f-8194-483b-8415-753e495ef448 kind: Event lastTimestamp: "2026-05-20T15:55:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:55:01Z" name: success-200-isvc-687f8.18b1510ce63fdaec namespace: kserve-ci-e2e-test resourceVersion: "20412" uid: 9fbc1bd9-cca7-4618-8095-f0293ced5439 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-687f8 namespace: kserve-ci-e2e-test resourceVersion: "19666" uid: 7c694c0f-8194-483b-8415-753e495ef448 kind: Event lastTimestamp: "2026-05-20T15:54:03Z" message: 'Failed to update status for InferenceService "success-200-isvc-687f8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-687f8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:54:03Z" name: success-200-isvc-687f8.18b1510ce6be2b98 namespace: kserve-ci-e2e-test resourceVersion: "19692" uid: da05822d-ca2e-4ca5-9df8-8e92c3597437 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-687f8 namespace: kserve-ci-e2e-test resourceVersion: "19666" uid: 7c694c0f-8194-483b-8415-753e495ef448 kind: Event lastTimestamp: "2026-05-20T15:54:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-687f8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:54:03Z" name: success-200-isvc-687f8.18b1510ce6be9749 namespace: kserve-ci-e2e-test resourceVersion: "19695" uid: de1d5a6f-7057-4171-8340-01cec26c2015 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-687f8 namespace: kserve-ci-e2e-test resourceVersion: "20410" uid: 7c694c0f-8194-483b-8415-753e495ef448 kind: Event lastTimestamp: "2026-05-20T15:55:01Z" message: InferenceService [success-200-isvc-687f8] is Ready metadata: creationTimestamp: "2026-05-20T15:55:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:55:01Z" name: success-200-isvc-687f8.18b1511a5d019078 namespace: kserve-ci-e2e-test resourceVersion: "20411" uid: fa070aff-fb03-4195-9dc6-a5f7984d1881 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:55:39.800878Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-80b6e-predictor-789b9765b-ldvnz namespace: kserve-ci-e2e-test resourceVersion: "21025" uid: 62f6ac5b-7653-47eb-a82d-c64312fde5ee kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-80b6e-predictor-789b9765b-ldvnz to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T15:55:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:55:39Z" name: success-200-isvc-80b6e-predictor-789b9765b-ldvnz.18b151234a559b9a namespace: kserve-ci-e2e-test resourceVersion: "21034" uid: 162c3087-a0dc-4b10-b118-8ad53882e124 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:40Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-80b6e-predictor-789b9765b-ldvnz namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: 62f6ac5b-7653-47eb-a82d-c64312fde5ee kind: Event lastTimestamp: "2026-05-20T15:55:40Z" message: Add eth0 [10.132.0.54/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:55:40Z" name: success-200-isvc-80b6e-predictor-789b9765b-ldvnz.18b151236439873a namespace: kserve-ci-e2e-test resourceVersion: "21076" uid: 3eda014c-237b-45cd-9886-845f4947db18 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-80b6e-predictor-789b9765b-ldvnz namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: 62f6ac5b-7653-47eb-a82d-c64312fde5ee kind: Event lastTimestamp: "2026-05-20T15:55:40Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-05-20T15:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:40Z" name: success-200-isvc-80b6e-predictor-789b9765b-ldvnz.18b1512365ca7f7c namespace: kserve-ci-e2e-test resourceVersion: "21078" uid: 4fe98136-03ef-4357-bfd3-dea1cf0750f0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-80b6e-predictor-789b9765b-ldvnz namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: 62f6ac5b-7653-47eb-a82d-c64312fde5ee kind: Event lastTimestamp: "2026-05-20T15:55:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:40Z" name: success-200-isvc-80b6e-predictor-789b9765b-ldvnz.18b151236a222d1d namespace: kserve-ci-e2e-test resourceVersion: "21079" uid: 3289c9d6-6a64-4d4f-9a8c-91eb9242ae54 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-80b6e-predictor-789b9765b-ldvnz namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: 62f6ac5b-7653-47eb-a82d-c64312fde5ee kind: Event lastTimestamp: "2026-05-20T15:55:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:40Z" name: success-200-isvc-80b6e-predictor-789b9765b-ldvnz.18b151236ab18dcf namespace: kserve-ci-e2e-test resourceVersion: "21080" uid: 0ab89a0a-fd04-457a-b629-fc00b3d3f00f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-80b6e-predictor-789b9765b-ldvnz namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: 62f6ac5b-7653-47eb-a82d-c64312fde5ee kind: Event lastTimestamp: "2026-05-20T15:55:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T15:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:40Z" name: success-200-isvc-80b6e-predictor-789b9765b-ldvnz.18b151236ac93df2 namespace: kserve-ci-e2e-test resourceVersion: "21081" uid: 37328162-a079-4aad-a547-c5982558d22a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-80b6e-predictor-789b9765b-ldvnz namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: 62f6ac5b-7653-47eb-a82d-c64312fde5ee kind: Event lastTimestamp: "2026-05-20T15:55:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:40Z" name: success-200-isvc-80b6e-predictor-789b9765b-ldvnz.18b151236e472a82 namespace: kserve-ci-e2e-test resourceVersion: "21085" uid: d74a9e1d-93b2-487c-ad42-16fd648e01fb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-80b6e-predictor-789b9765b-ldvnz namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: 62f6ac5b-7653-47eb-a82d-c64312fde5ee kind: Event lastTimestamp: "2026-05-20T15:55:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:55:40Z" name: success-200-isvc-80b6e-predictor-789b9765b-ldvnz.18b151236ec2604a namespace: kserve-ci-e2e-test resourceVersion: "21086" uid: 37246592-b4a8-403d-8840-27553c0b90e7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-20T15:55:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-80b6e-predictor-789b9765b-ldvnz namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: 62f6ac5b-7653-47eb-a82d-c64312fde5ee kind: Event lastTimestamp: "2026-05-20T16:05:07Z" message: 'Readiness probe failed: dial tcp 10.132.0.54:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:55:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:07Z" name: success-200-isvc-80b6e-predictor-789b9765b-ldvnz.18b15123bf331fa8 namespace: kserve-ci-e2e-test resourceVersion: "25753" uid: d6044a42-93ee-4049-add0-c9c448568ba3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-80b6e-predictor-789b9765b-ldvnz namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: 62f6ac5b-7653-47eb-a82d-c64312fde5ee kind: Event lastTimestamp: "2026-05-20T16:05:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T16:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:04Z" name: success-200-isvc-80b6e-predictor-789b9765b-ldvnz.18b151a6d19c29c3 namespace: kserve-ci-e2e-test resourceVersion: "25650" uid: 37c9e3d1-c4d4-465b-9fdd-ca6c69b0852e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-80b6e-predictor-789b9765b-ldvnz namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: 62f6ac5b-7653-47eb-a82d-c64312fde5ee kind: Event lastTimestamp: "2026-05-20T16:05:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T16:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:04Z" name: success-200-isvc-80b6e-predictor-789b9765b-ldvnz.18b151a6d19cd81d namespace: kserve-ci-e2e-test resourceVersion: "25652" uid: f6cf0070-7843-4319-a8cb-e6f88ee5eee9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-80b6e-predictor-789b9765b-ldvnz namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: 62f6ac5b-7653-47eb-a82d-c64312fde5ee kind: Event lastTimestamp: "2026-05-20T16:05:07Z" message: 'Readiness probe failed: Get "https://10.132.0.54:8643/healthz": dial tcp 10.132.0.54:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T16:05:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:07Z" name: success-200-isvc-80b6e-predictor-789b9765b-ldvnz.18b151a787c772b0 namespace: kserve-ci-e2e-test resourceVersion: "25751" uid: 73cc8b5f-678b-4cff-b94d-3a10bb5ee96d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-80b6e-predictor-789b9765b namespace: kserve-ci-e2e-test resourceVersion: "21008" uid: 1754b4fe-0858-4896-b9c0-4779cedb59bb kind: Event lastTimestamp: "2026-05-20T15:55:39Z" message: 'Created pod: success-200-isvc-80b6e-predictor-789b9765b-ldvnz' metadata: creationTimestamp: "2026-05-20T15:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:55:39Z" name: success-200-isvc-80b6e-predictor-789b9765b.18b15123494911f9 namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: 1660eaca-eb87-4b44-bedd-5a1c6a1f3a8d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:55:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-80b6e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21005" uid: 06b47db5-fd37-40f0-ba73-b764f8412c6a kind: Event lastTimestamp: "2026-05-20T15:55:39Z" message: Scaled up replica set success-200-isvc-80b6e-predictor-789b9765b from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:55:39Z" name: success-200-isvc-80b6e-predictor.18b1512346700277 namespace: kserve-ci-e2e-test resourceVersion: "21015" uid: 910b0f1f-beca-4ccd-9cf4-2dd37193c5f4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:55:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-80b6e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: fc7a6914-c8d7-4bcf-85ac-bf34b22e044a kind: Event lastTimestamp: "2026-05-20T15:56:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:55:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:56:24Z" name: success-200-isvc-80b6e-predictor.18b15126c4fd2292 namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 5a584f65-58d7-4702-b17b-eda2dc43202f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T15:55:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-80b6e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: fc7a6914-c8d7-4bcf-85ac-bf34b22e044a kind: Event lastTimestamp: "2026-05-20T15:56:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:55:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:56:24Z" name: success-200-isvc-80b6e-predictor.18b15126c552a2f9 namespace: kserve-ci-e2e-test resourceVersion: "21487" uid: 89409758-94fd-4097-a06d-4e179afec5bb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:56:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-80b6e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21231" uid: fc7a6914-c8d7-4bcf-85ac-bf34b22e044a kind: Event lastTimestamp: "2026-05-20T15:56:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:56:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:56:54Z" name: success-200-isvc-80b6e-predictor.18b151314254a0e4 namespace: kserve-ci-e2e-test resourceVersion: "21812" uid: dadab180-b0e1-4222-82dd-ebec5deed016 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T15:56:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-80b6e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21231" uid: fc7a6914-c8d7-4bcf-85ac-bf34b22e044a kind: Event lastTimestamp: "2026-05-20T15:56:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:56:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:56:54Z" name: success-200-isvc-80b6e-predictor.18b1513142c6f7a3 namespace: kserve-ci-e2e-test resourceVersion: "21813" uid: ca4ff9a7-d5ba-4a39-bd55-de4c65da85ad reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T15:55:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-80b6e namespace: kserve-ci-e2e-test resourceVersion: "20981" uid: bc5aeccb-8dd0-4392-ad70-0fb50501c259 kind: Event lastTimestamp: "2026-05-20T15:56:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:56:37Z" name: success-200-isvc-80b6e.18b1512346279283 namespace: kserve-ci-e2e-test resourceVersion: "21634" uid: a0ab3ec5-6530-4a78-a99f-a36f4bb0cf2e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:56:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-80b6e namespace: kserve-ci-e2e-test resourceVersion: "21628" uid: bc5aeccb-8dd0-4392-ad70-0fb50501c259 kind: Event lastTimestamp: "2026-05-20T15:56:37Z" message: InferenceService [success-200-isvc-80b6e] is Ready metadata: creationTimestamp: "2026-05-20T15:56:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:56:37Z" name: success-200-isvc-80b6e.18b15130cfb13f72 namespace: kserve-ci-e2e-test resourceVersion: "21629" uid: 6c225161-5be8-42b7-81ae-ea412b59af6c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:44:38.903057Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d4d66-predictor-54ddf55d4b-67mdg namespace: kserve-ci-e2e-test resourceVersion: "15267" uid: 1a538143-ec1a-49c3-9204-e35cf7adbd2e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d4d66-predictor-54ddf55d4b-67mdg to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T15:44:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:44:38Z" name: success-200-isvc-d4d66-predictor-54ddf55d4b-67mdg.18b1508969c1270c namespace: kserve-ci-e2e-test resourceVersion: "15275" uid: ab6f5644-c53f-4297-aec3-3fb5e872acd3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:39Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d4d66-predictor-54ddf55d4b-67mdg namespace: kserve-ci-e2e-test resourceVersion: "15278" uid: 1a538143-ec1a-49c3-9204-e35cf7adbd2e kind: Event lastTimestamp: "2026-05-20T15:44:39Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:44:39Z" name: success-200-isvc-d4d66-predictor-54ddf55d4b-67mdg.18b1508984482690 namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: 9427ce3f-3495-413d-ba6d-7c28d24a0190 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d4d66-predictor-54ddf55d4b-67mdg namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 1a538143-ec1a-49c3-9204-e35cf7adbd2e kind: Event lastTimestamp: "2026-05-20T15:44:39Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-05-20T15:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:39Z" name: success-200-isvc-d4d66-predictor-54ddf55d4b-67mdg.18b1508987294680 namespace: kserve-ci-e2e-test resourceVersion: "15318" uid: 93800093-e70e-4006-803b-c551ebede217 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d4d66-predictor-54ddf55d4b-67mdg namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 1a538143-ec1a-49c3-9204-e35cf7adbd2e kind: Event lastTimestamp: "2026-05-20T15:44:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T15:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:39Z" name: success-200-isvc-d4d66-predictor-54ddf55d4b-67mdg.18b1508989fd4ad2 namespace: kserve-ci-e2e-test resourceVersion: "15319" uid: c3fabb2e-cc9b-4fd6-b9f7-da2ed2eb23c4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d4d66-predictor-54ddf55d4b-67mdg namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 1a538143-ec1a-49c3-9204-e35cf7adbd2e kind: Event lastTimestamp: "2026-05-20T15:44:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T15:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:39Z" name: success-200-isvc-d4d66-predictor-54ddf55d4b-67mdg.18b150898a84f4cf namespace: kserve-ci-e2e-test resourceVersion: "15320" uid: 6b7941ce-8523-4b8a-842b-2042f41f426a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d4d66-predictor-54ddf55d4b-67mdg namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 1a538143-ec1a-49c3-9204-e35cf7adbd2e kind: Event lastTimestamp: "2026-05-20T15:44:39Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T15:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:39Z" name: success-200-isvc-d4d66-predictor-54ddf55d4b-67mdg.18b150898a92c1f6 namespace: kserve-ci-e2e-test resourceVersion: "15321" uid: deaf566f-4472-4cd9-8614-213fe7b88c55 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d4d66-predictor-54ddf55d4b-67mdg namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 1a538143-ec1a-49c3-9204-e35cf7adbd2e kind: Event lastTimestamp: "2026-05-20T15:44:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T15:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:39Z" name: success-200-isvc-d4d66-predictor-54ddf55d4b-67mdg.18b150898d558734 namespace: kserve-ci-e2e-test resourceVersion: "15322" uid: 31de6fc7-eef2-4475-a254-b5110d434d96 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d4d66-predictor-54ddf55d4b-67mdg namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 1a538143-ec1a-49c3-9204-e35cf7adbd2e kind: Event lastTimestamp: "2026-05-20T15:44:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:39Z" name: success-200-isvc-d4d66-predictor-54ddf55d4b-67mdg.18b150898dc27acf namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 93b9f685-ee72-4546-8cae-271e75c51a16 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-20T15:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d4d66-predictor-54ddf55d4b-67mdg namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 1a538143-ec1a-49c3-9204-e35cf7adbd2e kind: Event lastTimestamp: "2026-05-20T15:54:06Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:06Z" name: success-200-isvc-d4d66-predictor-54ddf55d4b-67mdg.18b15089be3dcdff namespace: kserve-ci-e2e-test resourceVersion: "19799" uid: e48c09e1-45d2-45b0-8c2f-5fe9ef554436 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d4d66-predictor-54ddf55d4b-67mdg namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 1a538143-ec1a-49c3-9204-e35cf7adbd2e kind: Event lastTimestamp: "2026-05-20T15:54:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T15:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:03Z" name: success-200-isvc-d4d66-predictor-54ddf55d4b-67mdg.18b1510ce5b23e10 namespace: kserve-ci-e2e-test resourceVersion: "19687" uid: 0da3dd74-448e-4306-b585-4b1f0af68c28 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d4d66-predictor-54ddf55d4b-67mdg namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 1a538143-ec1a-49c3-9204-e35cf7adbd2e kind: Event lastTimestamp: "2026-05-20T15:54:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T15:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:03Z" name: success-200-isvc-d4d66-predictor-54ddf55d4b-67mdg.18b1510ce5b2f034 namespace: kserve-ci-e2e-test resourceVersion: "19690" uid: 7ec535c2-a88e-4adc-a868-1be93157114b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d4d66-predictor-54ddf55d4b-67mdg namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 1a538143-ec1a-49c3-9204-e35cf7adbd2e kind: Event lastTimestamp: "2026-05-20T15:54:06Z" message: 'Readiness probe failed: Get "https://10.132.0.39:8643/healthz": dial tcp 10.132.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T15:54:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:06Z" name: success-200-isvc-d4d66-predictor-54ddf55d4b-67mdg.18b1510d86c218e6 namespace: kserve-ci-e2e-test resourceVersion: "19798" uid: 5728c5cd-71ef-42ae-8f4c-99549dea50e1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d4d66-predictor-54ddf55d4b namespace: kserve-ci-e2e-test resourceVersion: "15247" uid: 5363bdda-42d1-4bce-90c1-e63f0527212f kind: Event lastTimestamp: "2026-05-20T15:44:38Z" message: 'Created pod: success-200-isvc-d4d66-predictor-54ddf55d4b-67mdg' metadata: creationTimestamp: "2026-05-20T15:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:38Z" name: success-200-isvc-d4d66-predictor-54ddf55d4b.18b1508968a2d3ba namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: eab3a86d-5e39-4c2e-91f0-adf79b2ebf8a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d4d66-predictor namespace: kserve-ci-e2e-test resourceVersion: "15245" uid: 77c884b1-888f-4113-9145-fce2f628158e kind: Event lastTimestamp: "2026-05-20T15:44:38Z" message: Scaled up replica set success-200-isvc-d4d66-predictor-54ddf55d4b from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:38Z" name: success-200-isvc-d4d66-predictor.18b15089619964e6 namespace: kserve-ci-e2e-test resourceVersion: "15250" uid: fb79ab06-8d14-41fa-937b-4e8b8f557243 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:44:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d4d66-predictor namespace: kserve-ci-e2e-test resourceVersion: "15246" uid: b7b4e58a-8846-4e50-89a7-b304b69f1b62 kind: Event lastTimestamp: "2026-05-20T15:45:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:38Z" name: success-200-isvc-d4d66-predictor.18b1508ce02d2bd2 namespace: kserve-ci-e2e-test resourceVersion: "16066" uid: 166a80fe-cc68-41cf-815d-463365e43fb0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T15:44:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d4d66-predictor namespace: kserve-ci-e2e-test resourceVersion: "15246" uid: b7b4e58a-8846-4e50-89a7-b304b69f1b62 kind: Event lastTimestamp: "2026-05-20T15:45:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:38Z" name: success-200-isvc-d4d66-predictor.18b1508ce0aba055 namespace: kserve-ci-e2e-test resourceVersion: "16067" uid: 4cf1756e-51f4-43f7-aea9-a088a559b015 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d4d66-predictor namespace: kserve-ci-e2e-test resourceVersion: "15479" uid: b7b4e58a-8846-4e50-89a7-b304b69f1b62 kind: Event lastTimestamp: "2026-05-20T15:45:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:53Z" name: success-200-isvc-d4d66-predictor.18b1509ade0ee3f1 namespace: kserve-ci-e2e-test resourceVersion: "16222" uid: 7b5deb3c-972d-4a1e-a40d-9862be3dfb4b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d4d66-predictor namespace: kserve-ci-e2e-test resourceVersion: "15479" uid: b7b4e58a-8846-4e50-89a7-b304b69f1b62 kind: Event lastTimestamp: "2026-05-20T15:45:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:45:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:53Z" name: success-200-isvc-d4d66-predictor.18b1509adef76789 namespace: kserve-ci-e2e-test resourceVersion: "16223" uid: c4012918-e490-4757-bb4b-515eaafa947a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T15:44:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d4d66 namespace: kserve-ci-e2e-test resourceVersion: "15222" uid: f51d91b0-2c75-4aab-b615-e52359b793ad kind: Event lastTimestamp: "2026-05-20T15:45:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T15:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:45:36Z" name: success-200-isvc-d4d66.18b1508961583bb1 namespace: kserve-ci-e2e-test resourceVersion: "16042" uid: 9f04ecdb-3b7a-4b83-9177-e8485cb6101b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d4d66 namespace: kserve-ci-e2e-test resourceVersion: "15222" uid: f51d91b0-2c75-4aab-b615-e52359b793ad kind: Event lastTimestamp: "2026-05-20T15:44:38Z" message: 'Failed to update status for InferenceService "success-200-isvc-d4d66": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d4d66": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:44:38Z" name: success-200-isvc-d4d66.18b1508961d195e8 namespace: kserve-ci-e2e-test resourceVersion: "15249" uid: 561fe487-74de-47e6-a96d-af70713a3c40 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d4d66 namespace: kserve-ci-e2e-test resourceVersion: "15222" uid: f51d91b0-2c75-4aab-b615-e52359b793ad kind: Event lastTimestamp: "2026-05-20T15:44:38Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d4d66": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T15:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:44:38Z" name: success-200-isvc-d4d66.18b1508961d21bc1 namespace: kserve-ci-e2e-test resourceVersion: "15252" uid: dc4167b2-5111-4889-846b-4d37e2bc0994 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d4d66 namespace: kserve-ci-e2e-test resourceVersion: "16039" uid: f51d91b0-2c75-4aab-b615-e52359b793ad kind: Event lastTimestamp: "2026-05-20T15:45:36Z" message: InferenceService [success-200-isvc-d4d66] is Ready metadata: creationTimestamp: "2026-05-20T15:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:45:36Z" name: success-200-isvc-d4d66.18b15096cde2207b namespace: kserve-ci-e2e-test resourceVersion: "16040" uid: 6d10a558-4c2d-49b4-ace4-f49de559e7ad reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T16:04:48.968060Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq namespace: kserve-ci-e2e-test resourceVersion: "25387" uid: f4ac65ba-473b-4af6-bfc2-bce8092a55ce kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T16:04:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T16:04:48Z" name: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq.18b151a3273fd5b0 namespace: kserve-ci-e2e-test resourceVersion: "25395" uid: 68e0cdfd-1aa6-42df-b4a9-b2b1a19db200 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:49Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq namespace: kserve-ci-e2e-test resourceVersion: "25391" uid: f4ac65ba-473b-4af6-bfc2-bce8092a55ce kind: Event lastTimestamp: "2026-05-20T16:04:49Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-e05eb-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T16:04:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:04:49Z" name: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq.18b151a32ef587a1 namespace: kserve-ci-e2e-test resourceVersion: "25424" uid: 6aa8aa4b-41a0-496c-b1ec-2152a6a46801 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:50Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq namespace: kserve-ci-e2e-test resourceVersion: "25408" uid: f4ac65ba-473b-4af6-bfc2-bce8092a55ce kind: Event lastTimestamp: "2026-05-20T16:04:50Z" message: Add eth0 [10.132.0.57/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T16:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T16:04:50Z" name: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq.18b151a365a61727 namespace: kserve-ci-e2e-test resourceVersion: "25440" uid: 5b37b3c6-79e2-4656-a0b3-88018e3b23f8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq namespace: kserve-ci-e2e-test resourceVersion: "25391" uid: f4ac65ba-473b-4af6-bfc2-bce8092a55ce kind: Event lastTimestamp: "2026-05-20T16:04:50Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-05-20T16:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:04:50Z" name: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq.18b151a36710c53e namespace: kserve-ci-e2e-test resourceVersion: "25442" uid: 892d1413-0306-407a-a542-c694e9bb471d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq namespace: kserve-ci-e2e-test resourceVersion: "25391" uid: f4ac65ba-473b-4af6-bfc2-bce8092a55ce kind: Event lastTimestamp: "2026-05-20T16:04:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T16:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:04:50Z" name: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq.18b151a36a4375ca namespace: kserve-ci-e2e-test resourceVersion: "25447" uid: a5b65331-211e-445d-b4b7-6d1b8b6765da reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq namespace: kserve-ci-e2e-test resourceVersion: "25391" uid: f4ac65ba-473b-4af6-bfc2-bce8092a55ce kind: Event lastTimestamp: "2026-05-20T16:04:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T16:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:04:50Z" name: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq.18b151a36abf50d8 namespace: kserve-ci-e2e-test resourceVersion: "25449" uid: 07463db6-f1d3-4846-8f5c-8b64c88e56fd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq namespace: kserve-ci-e2e-test resourceVersion: "25391" uid: f4ac65ba-473b-4af6-bfc2-bce8092a55ce kind: Event lastTimestamp: "2026-05-20T16:04:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T16:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:04:50Z" name: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq.18b151a36accd92e namespace: kserve-ci-e2e-test resourceVersion: "25450" uid: ceb131d4-5be7-43e3-aeaa-eee0b129934e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq namespace: kserve-ci-e2e-test resourceVersion: "25391" uid: f4ac65ba-473b-4af6-bfc2-bce8092a55ce kind: Event lastTimestamp: "2026-05-20T16:04:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T16:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:04:50Z" name: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq.18b151a36f7dbd84 namespace: kserve-ci-e2e-test resourceVersion: "25456" uid: e62ef0d3-feb4-4de5-99cb-b6f8ed06ca3b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq namespace: kserve-ci-e2e-test resourceVersion: "25391" uid: f4ac65ba-473b-4af6-bfc2-bce8092a55ce kind: Event lastTimestamp: "2026-05-20T16:04:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T16:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:04:50Z" name: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq.18b151a36ff09880 namespace: kserve-ci-e2e-test resourceVersion: "25457" uid: b1cc1fee-a20b-4b43-8121-0d6f3c1982a2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-20T16:04:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq namespace: kserve-ci-e2e-test resourceVersion: "25391" uid: f4ac65ba-473b-4af6-bfc2-bce8092a55ce kind: Event lastTimestamp: "2026-05-20T16:05:36Z" message: 'Readiness probe failed: dial tcp 10.132.0.57:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T16:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:36Z" name: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq.18b151a38df289c2 namespace: kserve-ci-e2e-test resourceVersion: "26009" uid: 56e2625d-757a-4700-9aa4-62e914482981 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq namespace: kserve-ci-e2e-test resourceVersion: "25391" uid: f4ac65ba-473b-4af6-bfc2-bce8092a55ce kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:09Z" name: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq.18b151b5d9a51c92 namespace: kserve-ci-e2e-test resourceVersion: "26386" uid: 2fcb82b0-b146-466c-84e7-163d71f90cb5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq namespace: kserve-ci-e2e-test resourceVersion: "25391" uid: f4ac65ba-473b-4af6-bfc2-bce8092a55ce kind: Event lastTimestamp: "2026-05-20T16:06:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T16:06:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:09Z" name: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq.18b151b5d9a7ea7b namespace: kserve-ci-e2e-test resourceVersion: "26388" uid: 8d5e6c38-d3bf-4799-a6f9-3fd5bb02b4b4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq namespace: kserve-ci-e2e-test resourceVersion: "25391" uid: f4ac65ba-473b-4af6-bfc2-bce8092a55ce kind: Event lastTimestamp: "2026-05-20T16:06:11Z" message: 'Readiness probe failed: Get "https://10.132.0.57:8643/healthz": dial tcp 10.132.0.57:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T16:06:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:11Z" name: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq.18b151b66a24f2a8 namespace: kserve-ci-e2e-test resourceVersion: "26494" uid: 6799c3d9-b718-4668-8a82-179888c14548 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e05eb-predictor-d89f8c48d namespace: kserve-ci-e2e-test resourceVersion: "25362" uid: 45b61b2d-97ab-4b78-a997-c62dfb2d4cb3 kind: Event lastTimestamp: "2026-05-20T16:04:48Z" message: 'Created pod: success-200-isvc-e05eb-predictor-d89f8c48d-wrdnq' metadata: creationTimestamp: "2026-05-20T16:04:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:04:48Z" name: success-200-isvc-e05eb-predictor-d89f8c48d.18b151a326de298d namespace: kserve-ci-e2e-test resourceVersion: "25390" uid: 177e84c0-10d8-445e-9f66-7486a8ecf6a7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e05eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25358" uid: 29aa2414-12a9-45cf-a352-e6f453217232 kind: Event lastTimestamp: "2026-05-20T16:04:48Z" message: Scaled up replica set success-200-isvc-e05eb-predictor-d89f8c48d from 0 to 1 metadata: creationTimestamp: "2026-05-20T16:04:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:04:48Z" name: success-200-isvc-e05eb-predictor.18b151a322ca1e9f namespace: kserve-ci-e2e-test resourceVersion: "25366" uid: 7559999c-904b-453f-9350-089062d4de09 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T16:05:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e05eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25360" uid: 73b1d074-023a-4ac5-98d4-12a85ce86840 kind: Event lastTimestamp: "2026-05-20T16:05:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T16:05:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:05:33Z" name: success-200-isvc-e05eb-predictor.18b151a6a1cbdfd4 namespace: kserve-ci-e2e-test resourceVersion: "25980" uid: 7ab64a59-526c-4e3a-a46a-2635908dc4bc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T16:05:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e05eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25360" uid: 73b1d074-023a-4ac5-98d4-12a85ce86840 kind: Event lastTimestamp: "2026-05-20T16:05:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T16:05:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:05:33Z" name: success-200-isvc-e05eb-predictor.18b151a6a267951b namespace: kserve-ci-e2e-test resourceVersion: "25981" uid: b7b94e77-f96d-4db4-8917-08a5eab24656 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T16:05:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e05eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25574" uid: 73b1d074-023a-4ac5-98d4-12a85ce86840 kind: Event lastTimestamp: "2026-05-20T16:06:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T16:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:06:04Z" name: success-200-isvc-e05eb-predictor.18b151b1209277af namespace: kserve-ci-e2e-test resourceVersion: "26288" uid: 8caf2455-4256-42e0-aad0-e0e377c3e7d1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T16:05:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e05eb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25574" uid: 73b1d074-023a-4ac5-98d4-12a85ce86840 kind: Event lastTimestamp: "2026-05-20T16:06:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T16:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:06:04Z" name: success-200-isvc-e05eb-predictor.18b151b12104e960 namespace: kserve-ci-e2e-test resourceVersion: "26290" uid: af0e9248-7764-4632-8e6d-b0d2897c6b2d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T16:04:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e05eb namespace: kserve-ci-e2e-test resourceVersion: "25338" uid: 1e5b47f0-6784-4aab-9d52-4d253b0002fe kind: Event lastTimestamp: "2026-05-20T16:05:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T16:04:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:05:46Z" name: success-200-isvc-e05eb.18b151a3226901f9 namespace: kserve-ci-e2e-test resourceVersion: "26103" uid: 2a76aace-4da9-4459-970f-6314c850bffc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e05eb namespace: kserve-ci-e2e-test resourceVersion: "25338" uid: 1e5b47f0-6784-4aab-9d52-4d253b0002fe kind: Event lastTimestamp: "2026-05-20T16:04:48Z" message: 'Failed to update status for InferenceService "success-200-isvc-e05eb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e05eb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T16:04:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:04:48Z" name: success-200-isvc-e05eb.18b151a322e80e36 namespace: kserve-ci-e2e-test resourceVersion: "25369" uid: 04f052ad-ff94-4d8e-a1b7-cef4137f3802 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:04:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e05eb namespace: kserve-ci-e2e-test resourceVersion: "25338" uid: 1e5b47f0-6784-4aab-9d52-4d253b0002fe kind: Event lastTimestamp: "2026-05-20T16:04:48Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e05eb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T16:04:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:04:48Z" name: success-200-isvc-e05eb.18b151a322e87214 namespace: kserve-ci-e2e-test resourceVersion: "25370" uid: 4d1ccfbc-c9e6-4dbb-a858-09fa21f001c6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e05eb namespace: kserve-ci-e2e-test resourceVersion: "26101" uid: 1e5b47f0-6784-4aab-9d52-4d253b0002fe kind: Event lastTimestamp: "2026-05-20T16:05:46Z" message: InferenceService [success-200-isvc-e05eb] is Ready metadata: creationTimestamp: "2026-05-20T16:05:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:05:46Z" name: success-200-isvc-e05eb.18b151b09e4cd148 namespace: kserve-ci-e2e-test resourceVersion: "26102" uid: 4aa6cc8a-7b9e-42a6-af7b-c213dbb5adb2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T16:05:04.734220Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fd7d8-predictor-76dd8cbf6-7rp56 namespace: kserve-ci-e2e-test resourceVersion: "25653" uid: 722a5aa0-74ba-479e-91d4-e5db4b6b91e4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-fd7d8-predictor-76dd8cbf6-7rp56 to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T16:05:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T16:05:04Z" name: success-200-isvc-fd7d8-predictor-76dd8cbf6-7rp56.18b151a6d2fc589f namespace: kserve-ci-e2e-test resourceVersion: "25658" uid: e94a3f4d-a44f-4927-8932-9ac2ffee38e2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:04Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fd7d8-predictor-76dd8cbf6-7rp56 namespace: kserve-ci-e2e-test resourceVersion: "25656" uid: 722a5aa0-74ba-479e-91d4-e5db4b6b91e4 kind: Event lastTimestamp: "2026-05-20T16:05:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-fd7d8-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T16:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:04Z" name: success-200-isvc-fd7d8-predictor-76dd8cbf6-7rp56.18b151a6df5d68a0 namespace: kserve-ci-e2e-test resourceVersion: "25694" uid: 307c94d6-3a57-4603-9782-b3095ea840bb reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:05Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fd7d8-predictor-76dd8cbf6-7rp56 namespace: kserve-ci-e2e-test resourceVersion: "25663" uid: 722a5aa0-74ba-479e-91d4-e5db4b6b91e4 kind: Event lastTimestamp: "2026-05-20T16:05:05Z" message: Add eth0 [10.132.0.59/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T16:05:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T16:05:05Z" name: success-200-isvc-fd7d8-predictor-76dd8cbf6-7rp56.18b151a711607251 namespace: kserve-ci-e2e-test resourceVersion: "25717" uid: 5e804b3c-334f-4568-836c-d01caba3e47d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fd7d8-predictor-76dd8cbf6-7rp56 namespace: kserve-ci-e2e-test resourceVersion: "25656" uid: 722a5aa0-74ba-479e-91d4-e5db4b6b91e4 kind: Event lastTimestamp: "2026-05-20T16:05:05Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1433" already present on machine metadata: creationTimestamp: "2026-05-20T16:05:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:05Z" name: success-200-isvc-fd7d8-predictor-76dd8cbf6-7rp56.18b151a712ace6a2 namespace: kserve-ci-e2e-test resourceVersion: "25720" uid: 17113003-310f-4366-93ef-c76eff850eeb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fd7d8-predictor-76dd8cbf6-7rp56 namespace: kserve-ci-e2e-test resourceVersion: "25656" uid: 722a5aa0-74ba-479e-91d4-e5db4b6b91e4 kind: Event lastTimestamp: "2026-05-20T16:05:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T16:05:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:05Z" name: success-200-isvc-fd7d8-predictor-76dd8cbf6-7rp56.18b151a716c3d38f namespace: kserve-ci-e2e-test resourceVersion: "25724" uid: 55ca08e7-249b-4d4c-9937-f1ef6f55ce31 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fd7d8-predictor-76dd8cbf6-7rp56 namespace: kserve-ci-e2e-test resourceVersion: "25656" uid: 722a5aa0-74ba-479e-91d4-e5db4b6b91e4 kind: Event lastTimestamp: "2026-05-20T16:05:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T16:05:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:05Z" name: success-200-isvc-fd7d8-predictor-76dd8cbf6-7rp56.18b151a7174b460c namespace: kserve-ci-e2e-test resourceVersion: "25725" uid: 5e348bc2-0fce-40c7-8a09-1691411dd141 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fd7d8-predictor-76dd8cbf6-7rp56 namespace: kserve-ci-e2e-test resourceVersion: "25656" uid: 722a5aa0-74ba-479e-91d4-e5db4b6b91e4 kind: Event lastTimestamp: "2026-05-20T16:05:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T16:05:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:05Z" name: success-200-isvc-fd7d8-predictor-76dd8cbf6-7rp56.18b151a71759336d namespace: kserve-ci-e2e-test resourceVersion: "25726" uid: 7953a327-2d10-480f-b6f7-820556387924 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fd7d8-predictor-76dd8cbf6-7rp56 namespace: kserve-ci-e2e-test resourceVersion: "25656" uid: 722a5aa0-74ba-479e-91d4-e5db4b6b91e4 kind: Event lastTimestamp: "2026-05-20T16:05:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T16:05:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:05Z" name: success-200-isvc-fd7d8-predictor-76dd8cbf6-7rp56.18b151a71b0d1723 namespace: kserve-ci-e2e-test resourceVersion: "25727" uid: ae9d3263-70ed-4841-9835-84fb16416263 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fd7d8-predictor-76dd8cbf6-7rp56 namespace: kserve-ci-e2e-test resourceVersion: "25656" uid: 722a5aa0-74ba-479e-91d4-e5db4b6b91e4 kind: Event lastTimestamp: "2026-05-20T16:05:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T16:05:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:05Z" name: success-200-isvc-fd7d8-predictor-76dd8cbf6-7rp56.18b151a71b7b2791 namespace: kserve-ci-e2e-test resourceVersion: "25728" uid: 3d1e29a7-25bb-4d94-b6ba-4486e01b090f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-20T16:05:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fd7d8-predictor-76dd8cbf6-7rp56 namespace: kserve-ci-e2e-test resourceVersion: "25656" uid: 722a5aa0-74ba-479e-91d4-e5db4b6b91e4 kind: Event lastTimestamp: "2026-05-20T16:05:53Z" message: 'Readiness probe failed: dial tcp 10.132.0.59:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T16:05:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:05:53Z" name: success-200-isvc-fd7d8-predictor-76dd8cbf6-7rp56.18b151a787b3b11c namespace: kserve-ci-e2e-test resourceVersion: "26158" uid: 90b374fd-bc50-42ff-a0fd-90470e2eca13 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:22:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fd7d8-predictor-76dd8cbf6-7rp56 namespace: kserve-ci-e2e-test resourceVersion: "25656" uid: 722a5aa0-74ba-479e-91d4-e5db4b6b91e4 kind: Event lastTimestamp: "2026-05-20T16:22:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T16:22:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:22:34Z" name: success-200-isvc-fd7d8-predictor-76dd8cbf6-7rp56.18b1529b2e0e1524 namespace: kserve-ci-e2e-test resourceVersion: "33481" uid: b5e7ad8e-d0aa-4193-b6ca-92681abea446 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:22:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fd7d8-predictor-76dd8cbf6-7rp56 namespace: kserve-ci-e2e-test resourceVersion: "25656" uid: 722a5aa0-74ba-479e-91d4-e5db4b6b91e4 kind: Event lastTimestamp: "2026-05-20T16:22:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T16:22:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:22:34Z" name: success-200-isvc-fd7d8-predictor-76dd8cbf6-7rp56.18b1529b2e0ea9c9 namespace: kserve-ci-e2e-test resourceVersion: "33482" uid: 16722d07-368e-449d-87cd-ab392ae488b0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-fd7d8-predictor-76dd8cbf6 namespace: kserve-ci-e2e-test resourceVersion: "25641" uid: 09ffb4da-dd1c-467a-b64b-f26cc5bffa9a kind: Event lastTimestamp: "2026-05-20T16:05:04Z" message: 'Created pod: success-200-isvc-fd7d8-predictor-76dd8cbf6-7rp56' metadata: creationTimestamp: "2026-05-20T16:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:05:04Z" name: success-200-isvc-fd7d8-predictor-76dd8cbf6.18b151a6d2912f53 namespace: kserve-ci-e2e-test resourceVersion: "25655" uid: fd51b74c-8c4a-49f7-b16c-d4aa549bb307 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-fd7d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "25634" uid: 2618f8b8-7443-44a4-a713-afcc5f3b5fa3 kind: Event lastTimestamp: "2026-05-20T16:05:04Z" message: Scaled up replica set success-200-isvc-fd7d8-predictor-76dd8cbf6 from 0 to 1 metadata: creationTimestamp: "2026-05-20T16:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:05:04Z" name: success-200-isvc-fd7d8-predictor.18b151a6d0800ca8 namespace: kserve-ci-e2e-test resourceVersion: "25644" uid: 4ee1a27f-a329-4687-a578-05f4b4f09b25 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T16:05:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fd7d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "25640" uid: 785efcbb-27e5-4770-b7eb-ab74183474c8 kind: Event lastTimestamp: "2026-05-20T16:05:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T16:05:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:05:49Z" name: success-200-isvc-fd7d8-predictor.18b151aa4f1714bb namespace: kserve-ci-e2e-test resourceVersion: "26126" uid: 1614e729-1f9a-418b-bf78-2be063a6b031 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T16:05:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fd7d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "25640" uid: 785efcbb-27e5-4770-b7eb-ab74183474c8 kind: Event lastTimestamp: "2026-05-20T16:05:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T16:05:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:05:49Z" name: success-200-isvc-fd7d8-predictor.18b151aa4f7d6d29 namespace: kserve-ci-e2e-test resourceVersion: "26127" uid: 32b3f7d5-0e8b-420f-9e1a-33623538e737 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T16:06:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fd7d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "25868" uid: 785efcbb-27e5-4770-b7eb-ab74183474c8 kind: Event lastTimestamp: "2026-05-20T16:06:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T16:06:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:06:34Z" name: success-200-isvc-fd7d8-predictor.18b151b4ccc48448 namespace: kserve-ci-e2e-test resourceVersion: "26728" uid: d3372b87-b0f9-42ed-a4b7-306a8a5a06e7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T16:06:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fd7d8-predictor namespace: kserve-ci-e2e-test resourceVersion: "25868" uid: 785efcbb-27e5-4770-b7eb-ab74183474c8 kind: Event lastTimestamp: "2026-05-20T16:06:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T16:06:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:06:34Z" name: success-200-isvc-fd7d8-predictor.18b151b4cd3d06d7 namespace: kserve-ci-e2e-test resourceVersion: "26729" uid: 946f3583-acfc-4240-aaa0-d5fd291955fb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T16:05:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fd7d8 namespace: kserve-ci-e2e-test resourceVersion: "25605" uid: 3296f3da-04cf-44c7-b386-ef727e7c1d0b kind: Event lastTimestamp: "2026-05-20T16:06:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T16:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:06:03Z" name: success-200-isvc-fd7d8.18b151a6d05a53f4 namespace: kserve-ci-e2e-test resourceVersion: "26287" uid: f4585c64-d7c2-4576-b19c-3149417cc928 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fd7d8 namespace: kserve-ci-e2e-test resourceVersion: "25605" uid: 3296f3da-04cf-44c7-b386-ef727e7c1d0b kind: Event lastTimestamp: "2026-05-20T16:05:04Z" message: 'Failed to update status for InferenceService "success-200-isvc-fd7d8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-fd7d8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T16:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:05:04Z" name: success-200-isvc-fd7d8.18b151a6d0e8684b namespace: kserve-ci-e2e-test resourceVersion: "25648" uid: ed24e547-c6e0-40c4-9997-afb9dfa7b3cc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:05:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fd7d8 namespace: kserve-ci-e2e-test resourceVersion: "25605" uid: 3296f3da-04cf-44c7-b386-ef727e7c1d0b kind: Event lastTimestamp: "2026-05-20T16:05:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-fd7d8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T16:05:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:05:04Z" name: success-200-isvc-fd7d8.18b151a6d0e8af49 namespace: kserve-ci-e2e-test resourceVersion: "25649" uid: 6afe902b-9fd7-4535-b7b3-8bebf501fec4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fd7d8 namespace: kserve-ci-e2e-test resourceVersion: "26285" uid: 3296f3da-04cf-44c7-b386-ef727e7c1d0b kind: Event lastTimestamp: "2026-05-20T16:06:03Z" message: InferenceService [success-200-isvc-fd7d8] is Ready metadata: creationTimestamp: "2026-05-20T16:06:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:06:03Z" name: success-200-isvc-fd7d8.18b151b497eac5d4 namespace: kserve-ci-e2e-test resourceVersion: "26286" uid: eba2b14f-b9b6-43a7-8e9c-1732ab216a43 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:44:24.287354Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-542af-d57596c4d-5w6hk namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: b90118ab-d260-43ff-8b74-c492f22c2c6c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-542af-d57596c4d-5w6hk to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T15:44:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:44:24Z" name: switch-graph-542af-d57596c4d-5w6hk.18b1508602973558 namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: 8e483465-2945-47ac-a0a6-c57dfef79b41 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:24Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-542af-d57596c4d-5w6hk namespace: kserve-ci-e2e-test resourceVersion: "15038" uid: b90118ab-d260-43ff-8b74-c492f22c2c6c kind: Event lastTimestamp: "2026-05-20T15:44:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-542af-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T15:44:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:24Z" name: switch-graph-542af-d57596c4d-5w6hk.18b150860c40b2c0 namespace: kserve-ci-e2e-test resourceVersion: "15052" uid: dc47a49f-059a-4ae3-a433-0e05784c1209 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:25Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-542af-d57596c4d-5w6hk namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: b90118ab-d260-43ff-8b74-c492f22c2c6c kind: Event lastTimestamp: "2026-05-20T15:44:25Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:44:25Z" name: switch-graph-542af-d57596c4d-5w6hk.18b150863fd7e1b8 namespace: kserve-ci-e2e-test resourceVersion: "15064" uid: b7c9e3b9-819d-49bb-aaff-38a9c31cd425 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-542af} kind: Pod name: switch-graph-542af-d57596c4d-5w6hk namespace: kserve-ci-e2e-test resourceVersion: "15038" uid: b90118ab-d260-43ff-8b74-c492f22c2c6c kind: Event lastTimestamp: "2026-05-20T15:44:25Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:88955c235550cb8e8a9cbe850692d2683b6dea68a08743540305f46e732a59a3" metadata: creationTimestamp: "2026-05-20T15:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:25Z" name: switch-graph-542af-d57596c4d-5w6hk.18b15086411e8106 namespace: kserve-ci-e2e-test resourceVersion: "15066" uid: 187baaa0-cd06-4c57-99d1-a557aef820b0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-542af} kind: Pod name: switch-graph-542af-d57596c4d-5w6hk namespace: kserve-ci-e2e-test resourceVersion: "15038" uid: b90118ab-d260-43ff-8b74-c492f22c2c6c kind: Event lastTimestamp: "2026-05-20T15:44:27Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:88955c235550cb8e8a9cbe850692d2683b6dea68a08743540305f46e732a59a3" in 2.427s (2.427s including waiting). Image size: 218002168 bytes.' metadata: creationTimestamp: "2026-05-20T15:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:27Z" name: switch-graph-542af-d57596c4d-5w6hk.18b15086d1d1a568 namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: 0a06b7b4-21d0-449e-bc97-200ead5cbfd4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-542af} kind: Pod name: switch-graph-542af-d57596c4d-5w6hk namespace: kserve-ci-e2e-test resourceVersion: "15038" uid: b90118ab-d260-43ff-8b74-c492f22c2c6c kind: Event lastTimestamp: "2026-05-20T15:44:27Z" message: 'Created container: switch-graph-542af' metadata: creationTimestamp: "2026-05-20T15:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:27Z" name: switch-graph-542af-d57596c4d-5w6hk.18b15086d4c6c62d namespace: kserve-ci-e2e-test resourceVersion: "15090" uid: cab0302a-0ab4-4f50-bdee-eaf344603de7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-542af} kind: Pod name: switch-graph-542af-d57596c4d-5w6hk namespace: kserve-ci-e2e-test resourceVersion: "15038" uid: b90118ab-d260-43ff-8b74-c492f22c2c6c kind: Event lastTimestamp: "2026-05-20T15:44:27Z" message: Started container switch-graph-542af metadata: creationTimestamp: "2026-05-20T15:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:27Z" name: switch-graph-542af-d57596c4d-5w6hk.18b15086d53053a3 namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: ee3867a9-e4db-437b-ac86-5b06d2b70ad3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-542af} kind: Pod name: switch-graph-542af-d57596c4d-5w6hk namespace: kserve-ci-e2e-test resourceVersion: "15038" uid: b90118ab-d260-43ff-8b74-c492f22c2c6c kind: Event lastTimestamp: "2026-05-20T15:44:38Z" message: Stopping container switch-graph-542af metadata: creationTimestamp: "2026-05-20T15:44:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:44:38Z" name: switch-graph-542af-d57596c4d-5w6hk.18b1508951e3841f namespace: kserve-ci-e2e-test resourceVersion: "15217" uid: 02db11b9-9760-424e-95dc-4ac4d1d505c6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T15:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-542af} kind: Pod name: switch-graph-542af-d57596c4d-5w6hk namespace: kserve-ci-e2e-test resourceVersion: "15038" uid: b90118ab-d260-43ff-8b74-c492f22c2c6c kind: Event lastTimestamp: "2026-05-20T15:45:04Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T15:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:04Z" name: switch-graph-542af-d57596c4d-5w6hk.18b1508980458e3b namespace: kserve-ci-e2e-test resourceVersion: "15573" uid: f2276466-7b62-4e9d-a849-884343bfea18 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-542af-d57596c4d namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 1ccb0e2c-533b-413d-8bef-85a0269d0149 kind: Event lastTimestamp: "2026-05-20T15:44:24Z" message: 'Created pod: switch-graph-542af-d57596c4d-5w6hk' metadata: creationTimestamp: "2026-05-20T15:44:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:24Z" name: switch-graph-542af-d57596c4d.18b1508601c5c0de namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: 0fa9097c-6d7a-4f8d-8196-7ff28cf3f2eb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-542af namespace: kserve-ci-e2e-test resourceVersion: "15031" uid: c2200ad8-b1f2-4f73-963e-8e8377313dd8 kind: Event lastTimestamp: "2026-05-20T15:44:24Z" message: Scaled up replica set switch-graph-542af-d57596c4d from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:44:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:44:24Z" name: switch-graph-542af.18b1508600dabfd2 namespace: kserve-ci-e2e-test resourceVersion: "15035" uid: a0fcebf5-889d-4aa2-9213-d3d7ada9b29b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:44:24Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-542af namespace: kserve-ci-e2e-test resourceVersion: "15047" uid: 2e3a9b54-2390-4603-916b-62b931d8ad4d kind: Event lastTimestamp: "2026-05-20T15:44:24Z" message: InferenceGraph [switch-graph-542af] is Ready metadata: creationTimestamp: "2026-05-20T15:44:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:44:24Z" name: switch-graph-542af.18b150860818826d namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 30298845-34d1-4fa6-9740-13c1a39f07e8 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T15:45:48.805967Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d4d66-58988c469-qj57c namespace: kserve-ci-e2e-test resourceVersion: "16162" uid: 82ba7204-c40a-4fdc-97de-6257e21c8155 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-d4d66-58988c469-qj57c to ip-10-0-129-117.ec2.internal metadata: creationTimestamp: "2026-05-20T15:45:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T15:45:48Z" name: switch-graph-d4d66-58988c469-qj57c.18b15099b04af2a0 namespace: kserve-ci-e2e-test resourceVersion: "16168" uid: 1969c478-878e-4ced-83c7-636a2a3cb6c0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:49Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d4d66-58988c469-qj57c namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: 82ba7204-c40a-4fdc-97de-6257e21c8155 kind: Event lastTimestamp: "2026-05-20T15:45:49Z" message: Add eth0 [10.132.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T15:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T15:45:49Z" name: switch-graph-d4d66-58988c469-qj57c.18b15099cac07244 namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: 0330acc1-2e12-4b72-afc0-cecf4c1d4e79 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d4d66} kind: Pod name: switch-graph-d4d66-58988c469-qj57c namespace: kserve-ci-e2e-test resourceVersion: "16164" uid: 82ba7204-c40a-4fdc-97de-6257e21c8155 kind: Event lastTimestamp: "2026-05-20T15:45:49Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:88955c235550cb8e8a9cbe850692d2683b6dea68a08743540305f46e732a59a3" already present on machine metadata: creationTimestamp: "2026-05-20T15:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:49Z" name: switch-graph-d4d66-58988c469-qj57c.18b15099cc15022c namespace: kserve-ci-e2e-test resourceVersion: "16184" uid: 31a66055-d501-41a5-97fc-aff1ef47efcd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d4d66} kind: Pod name: switch-graph-d4d66-58988c469-qj57c namespace: kserve-ci-e2e-test resourceVersion: "16164" uid: 82ba7204-c40a-4fdc-97de-6257e21c8155 kind: Event lastTimestamp: "2026-05-20T15:45:49Z" message: 'Created container: switch-graph-d4d66' metadata: creationTimestamp: "2026-05-20T15:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:49Z" name: switch-graph-d4d66-58988c469-qj57c.18b15099cf6a72c9 namespace: kserve-ci-e2e-test resourceVersion: "16187" uid: 7124ed34-711e-4844-bab4-992d48c177c5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d4d66} kind: Pod name: switch-graph-d4d66-58988c469-qj57c namespace: kserve-ci-e2e-test resourceVersion: "16164" uid: 82ba7204-c40a-4fdc-97de-6257e21c8155 kind: Event lastTimestamp: "2026-05-20T15:45:49Z" message: Started container switch-graph-d4d66 metadata: creationTimestamp: "2026-05-20T15:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:45:49Z" name: switch-graph-d4d66-58988c469-qj57c.18b15099cfdd8a2f namespace: kserve-ci-e2e-test resourceVersion: "16188" uid: 7b5d0d7d-2549-474a-bf96-f5236a64f516 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:54:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d4d66} kind: Pod name: switch-graph-d4d66-58988c469-qj57c namespace: kserve-ci-e2e-test resourceVersion: "16164" uid: 82ba7204-c40a-4fdc-97de-6257e21c8155 kind: Event lastTimestamp: "2026-05-20T15:54:03Z" message: Stopping container switch-graph-d4d66 metadata: creationTimestamp: "2026-05-20T15:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:03Z" name: switch-graph-d4d66-58988c469-qj57c.18b1510cdf4446b0 namespace: kserve-ci-e2e-test resourceVersion: "19662" uid: 90ea49f9-383a-4271-9d29-0d8ecd30163a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T15:54:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d4d66} kind: Pod name: switch-graph-d4d66-58988c469-qj57c namespace: kserve-ci-e2e-test resourceVersion: "16164" uid: 82ba7204-c40a-4fdc-97de-6257e21c8155 kind: Event lastTimestamp: "2026-05-20T15:54:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T15:54:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T15:54:30Z" name: switch-graph-d4d66-58988c469-qj57c.18b1510d5c030b9b namespace: kserve-ci-e2e-test resourceVersion: "20119" uid: 625c1f5a-f810-493f-a388-79b0d2acf86d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-117.ec2.internal source: component: kubelet host: ip-10-0-129-117.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-d4d66-58988c469 namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: 0580641b-17f7-4e41-a6ba-49bf20ca881b kind: Event lastTimestamp: "2026-05-20T15:45:48Z" message: 'Created pod: switch-graph-d4d66-58988c469-qj57c' metadata: creationTimestamp: "2026-05-20T15:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:48Z" name: switch-graph-d4d66-58988c469.18b15099afd05c6b namespace: kserve-ci-e2e-test resourceVersion: "16163" uid: 2dc53780-79c8-4ef8-ac76-dd5ee0e1cfa6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-d4d66 namespace: kserve-ci-e2e-test resourceVersion: "16156" uid: 3d8760f8-819c-4866-b7e8-04bbcbd0c415 kind: Event lastTimestamp: "2026-05-20T15:45:48Z" message: Scaled up replica set switch-graph-d4d66-58988c469 from 0 to 1 metadata: creationTimestamp: "2026-05-20T15:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:45:48Z" name: switch-graph-d4d66.18b15099aea52eea namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: a3de4810-5e66-4687-b368-fd3f5eaf18d3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:45:48Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-d4d66 namespace: kserve-ci-e2e-test resourceVersion: "16176" uid: d9246384-8680-4718-9a43-c4d4c81a9540 kind: Event lastTimestamp: "2026-05-20T15:45:48Z" message: InferenceGraph [switch-graph-d4d66] is Ready metadata: creationTimestamp: "2026-05-20T15:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T15:45:48Z" name: switch-graph-d4d66.18b15099b60240bd namespace: kserve-ci-e2e-test resourceVersion: "16178" uid: c4b2f061-4569-4354-991f-16bc6a3b458f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d4d66 namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 06637061-429d-44e0-b7dd-b8c178f89970 kind: Event lastTimestamp: "2026-05-20T15:46:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:46:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:46:03Z" name: switch-graph-d4d66.18b1509d2d3d8703 namespace: kserve-ci-e2e-test resourceVersion: "16319" uid: 24971f91-67e7-4027-a992-af116a5512b5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d4d66 namespace: kserve-ci-e2e-test resourceVersion: "16157" uid: 06637061-429d-44e0-b7dd-b8c178f89970 kind: Event lastTimestamp: "2026-05-20T15:46:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T15:46:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:46:03Z" name: switch-graph-d4d66.18b1509d2dac3c54 namespace: kserve-ci-e2e-test resourceVersion: "16321" uid: 94b0b6bb-da44-4fc2-b7a4-cbdf7ffd1117 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d4d66 namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 06637061-429d-44e0-b7dd-b8c178f89970 kind: Event lastTimestamp: "2026-05-20T15:46:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:46:18Z" name: switch-graph-d4d66.18b150a0aca0faf9 namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: 3fb38d12-8292-4780-b4b4-5826fadea15e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T15:46:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d4d66 namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 06637061-429d-44e0-b7dd-b8c178f89970 kind: Event lastTimestamp: "2026-05-20T15:46:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T15:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T15:46:18Z" name: switch-graph-d4d66.18b150a0acfe1e6f namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 210498d8-c0fd-4493-83bf-6df2080ab6e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-20T16:06:14.850085Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-fd7d8-5949c54b67-xjpls namespace: kserve-ci-e2e-test resourceVersion: "26537" uid: a25f8c77-2139-46a5-8076-c5d47ce1a644 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-fd7d8-5949c54b67-xjpls to ip-10-0-135-174.ec2.internal metadata: creationTimestamp: "2026-05-20T16:06:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T16:06:14Z" name: switch-graph-fd7d8-5949c54b67-xjpls.18b151b7263788e7 namespace: kserve-ci-e2e-test resourceVersion: "26543" uid: 9727e340-708c-49e7-b06b-e74768c41be3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7d965446d4-pdtql source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:15Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-fd7d8-5949c54b67-xjpls namespace: kserve-ci-e2e-test resourceVersion: "26540" uid: a25f8c77-2139-46a5-8076-c5d47ce1a644 kind: Event lastTimestamp: "2026-05-20T16:06:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-fd7d8-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T16:06:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:15Z" name: switch-graph-fd7d8-5949c54b67-xjpls.18b151b73306c255 namespace: kserve-ci-e2e-test resourceVersion: "26554" uid: e6d72a8f-2f5b-4cab-8832-34968dcc4a10 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:15Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-fd7d8-5949c54b67-xjpls namespace: kserve-ci-e2e-test resourceVersion: "26546" uid: a25f8c77-2139-46a5-8076-c5d47ce1a644 kind: Event lastTimestamp: "2026-05-20T16:06:15Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T16:06:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T16:06:15Z" name: switch-graph-fd7d8-5949c54b67-xjpls.18b151b76348acbe namespace: kserve-ci-e2e-test resourceVersion: "26563" uid: efb064c2-27db-4b93-b882-9d067f27b1d0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-fd7d8} kind: Pod name: switch-graph-fd7d8-5949c54b67-xjpls namespace: kserve-ci-e2e-test resourceVersion: "26540" uid: a25f8c77-2139-46a5-8076-c5d47ce1a644 kind: Event lastTimestamp: "2026-05-20T16:06:15Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:88955c235550cb8e8a9cbe850692d2683b6dea68a08743540305f46e732a59a3" already present on machine metadata: creationTimestamp: "2026-05-20T16:06:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:15Z" name: switch-graph-fd7d8-5949c54b67-xjpls.18b151b7652aa8dc namespace: kserve-ci-e2e-test resourceVersion: "26566" uid: 0929bdd0-5eab-4372-b1ff-cc1676f5a536 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-fd7d8} kind: Pod name: switch-graph-fd7d8-5949c54b67-xjpls namespace: kserve-ci-e2e-test resourceVersion: "26540" uid: a25f8c77-2139-46a5-8076-c5d47ce1a644 kind: Event lastTimestamp: "2026-05-20T16:06:15Z" message: 'Created container: switch-graph-fd7d8' metadata: creationTimestamp: "2026-05-20T16:06:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:15Z" name: switch-graph-fd7d8-5949c54b67-xjpls.18b151b767cc6795 namespace: kserve-ci-e2e-test resourceVersion: "26567" uid: db6086ee-1277-4b21-ba1c-f32350014889 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-fd7d8} kind: Pod name: switch-graph-fd7d8-5949c54b67-xjpls namespace: kserve-ci-e2e-test resourceVersion: "26540" uid: a25f8c77-2139-46a5-8076-c5d47ce1a644 kind: Event lastTimestamp: "2026-05-20T16:06:15Z" message: Started container switch-graph-fd7d8 metadata: creationTimestamp: "2026-05-20T16:06:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:06:15Z" name: switch-graph-fd7d8-5949c54b67-xjpls.18b151b7686caa92 namespace: kserve-ci-e2e-test resourceVersion: "26568" uid: 03de324e-1891-4c6e-a5e9-fc0935664d0c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:22:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-fd7d8} kind: Pod name: switch-graph-fd7d8-5949c54b67-xjpls namespace: kserve-ci-e2e-test resourceVersion: "26540" uid: a25f8c77-2139-46a5-8076-c5d47ce1a644 kind: Event lastTimestamp: "2026-05-20T16:22:34Z" message: Stopping container switch-graph-fd7d8 metadata: creationTimestamp: "2026-05-20T16:22:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:22:34Z" name: switch-graph-fd7d8-5949c54b67-xjpls.18b1529b26e9a3b1 namespace: kserve-ci-e2e-test resourceVersion: "33467" uid: f652f454-ca42-4ac8-b6af-2c2ec533d539 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T16:22:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-fd7d8} kind: Pod name: switch-graph-fd7d8-5949c54b67-xjpls namespace: kserve-ci-e2e-test resourceVersion: "26540" uid: a25f8c77-2139-46a5-8076-c5d47ce1a644 kind: Event lastTimestamp: "2026-05-20T16:22:57Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T16:22:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T16:22:57Z" name: switch-graph-fd7d8-5949c54b67-xjpls.18b1529bf83a52b6 namespace: kserve-ci-e2e-test resourceVersion: "33703" uid: c640fd2d-fb3f-471e-b9f9-4191acd7caa6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-174.ec2.internal source: component: kubelet host: ip-10-0-135-174.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-fd7d8-5949c54b67 namespace: kserve-ci-e2e-test resourceVersion: "26534" uid: f6b5206b-c38b-47ef-b3de-6bc0f1a67e55 kind: Event lastTimestamp: "2026-05-20T16:06:14Z" message: 'Created pod: switch-graph-fd7d8-5949c54b67-xjpls' metadata: creationTimestamp: "2026-05-20T16:06:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:06:14Z" name: switch-graph-fd7d8-5949c54b67.18b151b72541b849 namespace: kserve-ci-e2e-test resourceVersion: "26538" uid: 8e234dc4-5079-4ad5-9de7-2b6bf4721447 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-fd7d8 namespace: kserve-ci-e2e-test resourceVersion: "26532" uid: 2be44f83-ec15-440c-a87d-22be85360179 kind: Event lastTimestamp: "2026-05-20T16:06:14Z" message: Scaled up replica set switch-graph-fd7d8-5949c54b67 from 0 to 1 metadata: creationTimestamp: "2026-05-20T16:06:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:06:14Z" name: switch-graph-fd7d8.18b151b7241eeaf7 namespace: kserve-ci-e2e-test resourceVersion: "26535" uid: 9c09cb6f-77e9-4ec6-b526-a38d022a5bd7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T16:06:14Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-fd7d8 namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: c3a2e54d-dd45-4ddf-a89c-b2bf8561e3de kind: Event lastTimestamp: "2026-05-20T16:06:14Z" message: InferenceGraph [switch-graph-fd7d8] is Ready metadata: creationTimestamp: "2026-05-20T16:06:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T16:06:14Z" name: switch-graph-fd7d8.18b151b72d49c001 namespace: kserve-ci-e2e-test resourceVersion: "26552" uid: 4396c5a6-fe2c-45d4-a8a8-107182fe371f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T16:06:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-fd7d8 namespace: kserve-ci-e2e-test resourceVersion: "26533" uid: 4592358d-880d-40ce-a9e8-44400f9a654e kind: Event lastTimestamp: "2026-05-20T16:06:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T16:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:06:59Z" name: switch-graph-fd7d8.18b151baa28f8bef namespace: kserve-ci-e2e-test resourceVersion: "26939" uid: e7737240-4bac-4f86-a883-e384b2da6dce reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T16:06:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-fd7d8 namespace: kserve-ci-e2e-test resourceVersion: "26533" uid: 4592358d-880d-40ce-a9e8-44400f9a654e kind: Event lastTimestamp: "2026-05-20T16:06:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T16:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T16:06:59Z" name: switch-graph-fd7d8.18b151baa3002378 namespace: kserve-ci-e2e-test resourceVersion: "26940" uid: 101cd9f1-5672-47db-af69-5c8edc0d8577 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "33726"