--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:56:04.721809Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-1f879-76cbbb6865-l8c8g namespace: kserve-ci-e2e-test resourceVersion: "21499" uid: be147abb-e4e1-4eef-9ed1-1e4162a088ca kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-1f879-76cbbb6865-l8c8g to ip-10-0-132-182.ec2.internal metadata: creationTimestamp: "2026-07-08T16:56:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:56:04Z" name: ensemble-graph-1f879-76cbbb6865-l8c8g.18c05edf189482db namespace: kserve-ci-e2e-test resourceVersion: "21506" uid: 872f7f22-a5d5-4956-825b-c7c2030c169f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:04Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-1f879-76cbbb6865-l8c8g namespace: kserve-ci-e2e-test resourceVersion: "21502" uid: be147abb-e4e1-4eef-9ed1-1e4162a088ca kind: Event lastTimestamp: "2026-07-08T16:56:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-1f879-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T16:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:56:04Z" name: ensemble-graph-1f879-76cbbb6865-l8c8g.18c05edf225995bd namespace: kserve-ci-e2e-test resourceVersion: "21519" uid: 3c7c96b6-76b2-4e4a-8cea-67326b43cec3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:05Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-1f879-76cbbb6865-l8c8g namespace: kserve-ci-e2e-test resourceVersion: "21509" uid: be147abb-e4e1-4eef-9ed1-1e4162a088ca kind: Event lastTimestamp: "2026-07-08T16:56:05Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:56:05Z" name: ensemble-graph-1f879-76cbbb6865-l8c8g.18c05edf55de57c5 namespace: kserve-ci-e2e-test resourceVersion: "21531" uid: 20f19384-f759-4c90-b7fb-13948c63b4ab reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1f879} kind: Pod name: ensemble-graph-1f879-76cbbb6865-l8c8g namespace: kserve-ci-e2e-test resourceVersion: "21502" uid: be147abb-e4e1-4eef-9ed1-1e4162a088ca kind: Event lastTimestamp: "2026-07-08T16:56:05Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:86bfaabcc176433317a84dc02e835b9657d32290b963c2e4ae8312aa72bd9d64" already present on machine metadata: creationTimestamp: "2026-07-08T16:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:56:05Z" name: ensemble-graph-1f879-76cbbb6865-l8c8g.18c05edf571e6ef5 namespace: kserve-ci-e2e-test resourceVersion: "21533" uid: b980acf2-5361-4f43-b65a-44c4b8455c5c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1f879} kind: Pod name: ensemble-graph-1f879-76cbbb6865-l8c8g namespace: kserve-ci-e2e-test resourceVersion: "21502" uid: be147abb-e4e1-4eef-9ed1-1e4162a088ca kind: Event lastTimestamp: "2026-07-08T16:56:05Z" message: 'Created container: ensemble-graph-1f879' metadata: creationTimestamp: "2026-07-08T16:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:56:05Z" name: ensemble-graph-1f879-76cbbb6865-l8c8g.18c05edf5a5917dd namespace: kserve-ci-e2e-test resourceVersion: "21534" uid: a5cc790d-0fa1-48c9-acd4-f77bfe4d1235 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1f879} kind: Pod name: ensemble-graph-1f879-76cbbb6865-l8c8g namespace: kserve-ci-e2e-test resourceVersion: "21502" uid: be147abb-e4e1-4eef-9ed1-1e4162a088ca kind: Event lastTimestamp: "2026-07-08T16:56:05Z" message: Started container ensemble-graph-1f879 metadata: creationTimestamp: "2026-07-08T16:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:56:05Z" name: ensemble-graph-1f879-76cbbb6865-l8c8g.18c05edf5b2c8eb4 namespace: kserve-ci-e2e-test resourceVersion: "21535" uid: 15627a8c-5036-4f65-8640-a0abdadca48d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1f879} kind: Pod name: ensemble-graph-1f879-76cbbb6865-l8c8g namespace: kserve-ci-e2e-test resourceVersion: "21502" uid: be147abb-e4e1-4eef-9ed1-1e4162a088ca kind: Event lastTimestamp: "2026-07-08T17:04:19Z" message: Stopping container ensemble-graph-1f879 metadata: creationTimestamp: "2026-07-08T17:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:19Z" name: ensemble-graph-1f879-76cbbb6865-l8c8g.18c05f5240a16283 namespace: kserve-ci-e2e-test resourceVersion: "25295" uid: 208d4fd4-71e7-46ae-a6ac-1313f3a05662 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T17:04:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1f879} kind: Pod name: ensemble-graph-1f879-76cbbb6865-l8c8g namespace: kserve-ci-e2e-test resourceVersion: "21502" uid: be147abb-e4e1-4eef-9ed1-1e4162a088ca kind: Event lastTimestamp: "2026-07-08T17:04:47Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T17:04:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:47Z" name: ensemble-graph-1f879-76cbbb6865-l8c8g.18c05f53113bf205 namespace: kserve-ci-e2e-test resourceVersion: "25641" uid: d6e62cb2-67dd-4895-80e3-c8d76d19eebb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-1f879-76cbbb6865 namespace: kserve-ci-e2e-test resourceVersion: "21494" uid: f49afcf5-1d60-46e8-abb3-d9e6ff3fa015 kind: Event lastTimestamp: "2026-07-08T16:56:04Z" message: 'Created pod: ensemble-graph-1f879-76cbbb6865-l8c8g' metadata: creationTimestamp: "2026-07-08T16:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:56:04Z" name: ensemble-graph-1f879-76cbbb6865.18c05edf1805f894 namespace: kserve-ci-e2e-test resourceVersion: "21501" uid: 5f8a170d-9662-4bc2-9197-c9edb6755950 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-1f879 namespace: kserve-ci-e2e-test resourceVersion: "21492" uid: 97049cbf-89f9-46b4-9ca6-8d685cc48f34 kind: Event lastTimestamp: "2026-07-08T16:56:04Z" message: Scaled up replica set ensemble-graph-1f879-76cbbb6865 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:56:04Z" name: ensemble-graph-1f879.18c05edf1741d4fb namespace: kserve-ci-e2e-test resourceVersion: "21497" uid: b6334e34-67a9-48c3-b787-79d0b4e3bb25 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-1f879 namespace: kserve-ci-e2e-test resourceVersion: "21512" uid: b370adef-87d0-4912-9b9e-fb1fb7de1efb kind: Event lastTimestamp: "2026-07-08T16:56:04Z" message: InferenceGraph [ensemble-graph-1f879] is Ready metadata: creationTimestamp: "2026-07-08T16:56:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:56:04Z" name: ensemble-graph-1f879.18c05edf1d071339 namespace: kserve-ci-e2e-test resourceVersion: "21514" uid: 7df26ed8-c370-46e7-a86e-7ac2419eab39 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-1f879 namespace: kserve-ci-e2e-test resourceVersion: "21493" uid: 68b789b8-42af-41a3-8272-9c642c4e1efa kind: Event lastTimestamp: "2026-07-08T16:56:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:56:19Z" name: ensemble-graph-1f879.18c05ee295c3b4e8 namespace: kserve-ci-e2e-test resourceVersion: "21654" uid: 5fe42c9d-0695-4658-905e-c1d4785696db reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-1f879 namespace: kserve-ci-e2e-test resourceVersion: "21493" uid: 68b789b8-42af-41a3-8272-9c642c4e1efa kind: Event lastTimestamp: "2026-07-08T16:56:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:56:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:56:19Z" name: ensemble-graph-1f879.18c05ee2962ed7d4 namespace: kserve-ci-e2e-test resourceVersion: "21658" uid: 6ec06477-22cc-46a7-b617-457fb8571cf1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-1f879 namespace: kserve-ci-e2e-test resourceVersion: "21655" uid: 68b789b8-42af-41a3-8272-9c642c4e1efa kind: Event lastTimestamp: "2026-07-08T16:56:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:56:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:56:34Z" name: ensemble-graph-1f879.18c05ee614e9e90a namespace: kserve-ci-e2e-test resourceVersion: "21783" uid: 522fcb2f-9a8e-40ba-b8c0-02347e880f3d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-1f879 namespace: kserve-ci-e2e-test resourceVersion: "21655" uid: 68b789b8-42af-41a3-8272-9c642c4e1efa kind: Event lastTimestamp: "2026-07-08T16:56:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:56:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:56:34Z" name: ensemble-graph-1f879.18c05ee61552636a namespace: kserve-ci-e2e-test resourceVersion: "21785" uid: 060bab17-421a-4d56-be52-c4416d564c03 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:54:54.376970Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-8befa-c7fbffddd-k7n7q namespace: kserve-ci-e2e-test resourceVersion: "20521" uid: 50981f56-610f-4f03-848f-5a37da7b8ad7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-8befa-c7fbffddd-k7n7q to ip-10-0-132-182.ec2.internal metadata: creationTimestamp: "2026-07-08T16:54:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:54:54Z" name: ensemble-graph-8befa-c7fbffddd-k7n7q.18c05eceb7b36269 namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: ba1eca89-096e-4aaf-bccb-2d6808fb03b8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:54Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-8befa-c7fbffddd-k7n7q namespace: kserve-ci-e2e-test resourceVersion: "20530" uid: 50981f56-610f-4f03-848f-5a37da7b8ad7 kind: Event lastTimestamp: "2026-07-08T16:54:54Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:54:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:54:54Z" name: ensemble-graph-8befa-c7fbffddd-k7n7q.18c05eced222eeda namespace: kserve-ci-e2e-test resourceVersion: "20543" uid: 7fdb2ca9-9c5a-4014-b609-8350fe7d7bda reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-8befa} kind: Pod name: ensemble-graph-8befa-c7fbffddd-k7n7q namespace: kserve-ci-e2e-test resourceVersion: "20523" uid: 50981f56-610f-4f03-848f-5a37da7b8ad7 kind: Event lastTimestamp: "2026-07-08T16:54:54Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:86bfaabcc176433317a84dc02e835b9657d32290b963c2e4ae8312aa72bd9d64" already present on machine metadata: creationTimestamp: "2026-07-08T16:54:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:54Z" name: ensemble-graph-8befa-c7fbffddd-k7n7q.18c05eced39488a2 namespace: kserve-ci-e2e-test resourceVersion: "20545" uid: ed67393a-972c-4590-ae0c-cedf34ee848e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-8befa} kind: Pod name: ensemble-graph-8befa-c7fbffddd-k7n7q namespace: kserve-ci-e2e-test resourceVersion: "20523" uid: 50981f56-610f-4f03-848f-5a37da7b8ad7 kind: Event lastTimestamp: "2026-07-08T16:54:54Z" message: 'Created container: ensemble-graph-8befa' metadata: creationTimestamp: "2026-07-08T16:54:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:54Z" name: ensemble-graph-8befa-c7fbffddd-k7n7q.18c05eced714b30b namespace: kserve-ci-e2e-test resourceVersion: "20547" uid: 0d244e55-dc9b-4ad9-a124-e46fc4e81eda reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-8befa} kind: Pod name: ensemble-graph-8befa-c7fbffddd-k7n7q namespace: kserve-ci-e2e-test resourceVersion: "20523" uid: 50981f56-610f-4f03-848f-5a37da7b8ad7 kind: Event lastTimestamp: "2026-07-08T16:54:54Z" message: Started container ensemble-graph-8befa metadata: creationTimestamp: "2026-07-08T16:54:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:54Z" name: ensemble-graph-8befa-c7fbffddd-k7n7q.18c05eced78f6f0f namespace: kserve-ci-e2e-test resourceVersion: "20548" uid: 9c171d74-79ca-4cce-b058-b5dcf7efbf02 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-8befa} kind: Pod name: ensemble-graph-8befa-c7fbffddd-k7n7q namespace: kserve-ci-e2e-test resourceVersion: "20523" uid: 50981f56-610f-4f03-848f-5a37da7b8ad7 kind: Event lastTimestamp: "2026-07-08T16:55:04Z" message: Stopping container ensemble-graph-8befa metadata: creationTimestamp: "2026-07-08T16:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:04Z" name: ensemble-graph-8befa-c7fbffddd-k7n7q.18c05ed110e5ac2c namespace: kserve-ci-e2e-test resourceVersion: "20646" uid: f24f4cba-084f-4a20-add1-56814883eae7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:55:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-8befa} kind: Pod name: ensemble-graph-8befa-c7fbffddd-k7n7q namespace: kserve-ci-e2e-test resourceVersion: "20523" uid: 50981f56-610f-4f03-848f-5a37da7b8ad7 kind: Event lastTimestamp: "2026-07-08T16:55:31Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T16:55:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:31Z" name: ensemble-graph-8befa-c7fbffddd-k7n7q.18c05ed188b2bd6f namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: 93097f72-6ab5-4127-9283-7322b7c3a091 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-8befa-c7fbffddd namespace: kserve-ci-e2e-test resourceVersion: "20518" uid: bb4edf70-48c8-4ae0-b825-6fbbe9990400 kind: Event lastTimestamp: "2026-07-08T16:54:54Z" message: 'Created pod: ensemble-graph-8befa-c7fbffddd-k7n7q' metadata: creationTimestamp: "2026-07-08T16:54:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:54:54Z" name: ensemble-graph-8befa-c7fbffddd.18c05eceb75961af namespace: kserve-ci-e2e-test resourceVersion: "20522" uid: 749a4fea-544f-46d7-923a-eeb2c3d0528d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-8befa namespace: kserve-ci-e2e-test resourceVersion: "20516" uid: 81140e9d-8826-44bb-8193-dccc321d86d2 kind: Event lastTimestamp: "2026-07-08T16:54:54Z" message: Scaled up replica set ensemble-graph-8befa-c7fbffddd from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:54:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:54:54Z" name: ensemble-graph-8befa.18c05eceb6a4a97d namespace: kserve-ci-e2e-test resourceVersion: "20520" uid: 198840f5-e1a7-406d-9fac-379f0e0b585a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:54Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-8befa namespace: kserve-ci-e2e-test resourceVersion: "20536" uid: dd9f8c91-f8cd-405b-897a-945931e04b26 kind: Event lastTimestamp: "2026-07-08T16:54:54Z" message: InferenceGraph [ensemble-graph-8befa] is Ready metadata: creationTimestamp: "2026-07-08T16:54:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:54:54Z" name: ensemble-graph-8befa.18c05ecebc2b03b3 namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: f8d1361e-9a02-4cfc-b0c2-0b5d3ae59c49 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:55:04.797814Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1f879-predictor-5d845d546c-bwfkj namespace: kserve-ci-e2e-test resourceVersion: "20729" uid: bb734e6a-0f34-4b49-a994-9fbdda88f368 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1f879-predictor-5d845d546c-bwfkj to ip-10-0-138-233.ec2.internal metadata: creationTimestamp: "2026-07-08T16:55:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:55:04Z" name: error-404-isvc-1f879-predictor-5d845d546c-bwfkj.18c05ed124d4d11e namespace: kserve-ci-e2e-test resourceVersion: "20733" uid: b2f41f70-b5ac-413c-aa47-25fc41101847 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:05Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1f879-predictor-5d845d546c-bwfkj namespace: kserve-ci-e2e-test resourceVersion: "20739" uid: bb734e6a-0f34-4b49-a994-9fbdda88f368 kind: Event lastTimestamp: "2026-07-08T16:55:05Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:55:05Z" name: error-404-isvc-1f879-predictor-5d845d546c-bwfkj.18c05ed13e88a9f3 namespace: kserve-ci-e2e-test resourceVersion: "20756" uid: 562f2256-ff99-40fe-81de-7af73c438776 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1f879-predictor-5d845d546c-bwfkj namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: bb734e6a-0f34-4b49-a994-9fbdda88f368 kind: Event lastTimestamp: "2026-07-08T16:55:05Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1707" already present on machine metadata: creationTimestamp: "2026-07-08T16:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:05Z" name: error-404-isvc-1f879-predictor-5d845d546c-bwfkj.18c05ed13ff694da namespace: kserve-ci-e2e-test resourceVersion: "20760" uid: 636ea309-337e-4527-a61a-ef277b510966 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1f879-predictor-5d845d546c-bwfkj namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: bb734e6a-0f34-4b49-a994-9fbdda88f368 kind: Event lastTimestamp: "2026-07-08T16:55:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:05Z" name: error-404-isvc-1f879-predictor-5d845d546c-bwfkj.18c05ed142cf7e2b namespace: kserve-ci-e2e-test resourceVersion: "20763" uid: 47c79c9f-298d-4029-b41c-e518cdad4d07 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1f879-predictor-5d845d546c-bwfkj namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: bb734e6a-0f34-4b49-a994-9fbdda88f368 kind: Event lastTimestamp: "2026-07-08T16:55:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:05Z" name: error-404-isvc-1f879-predictor-5d845d546c-bwfkj.18c05ed143424366 namespace: kserve-ci-e2e-test resourceVersion: "20765" uid: f30068d4-bb9c-4cc4-b4c0-96142304d036 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1f879-predictor-5d845d546c-bwfkj namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: bb734e6a-0f34-4b49-a994-9fbdda88f368 kind: Event lastTimestamp: "2026-07-08T16:55:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:05Z" name: error-404-isvc-1f879-predictor-5d845d546c-bwfkj.18c05ed1434f417c namespace: kserve-ci-e2e-test resourceVersion: "20766" uid: 8bf1bda9-de88-42dc-9b57-8cab9d9b1bb4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1f879-predictor-5d845d546c-bwfkj namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: bb734e6a-0f34-4b49-a994-9fbdda88f368 kind: Event lastTimestamp: "2026-07-08T16:55:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:05Z" name: error-404-isvc-1f879-predictor-5d845d546c-bwfkj.18c05ed146062557 namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: a6776a08-3e70-4bbf-9ad5-b9d3927d57d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1f879-predictor-5d845d546c-bwfkj namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: bb734e6a-0f34-4b49-a994-9fbdda88f368 kind: Event lastTimestamp: "2026-07-08T16:55:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:05Z" name: error-404-isvc-1f879-predictor-5d845d546c-bwfkj.18c05ed1467b0680 namespace: kserve-ci-e2e-test resourceVersion: "20768" uid: 96a773d3-4d43-4e7f-a49c-51991caba0c1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-08T16:55:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1f879-predictor-5d845d546c-bwfkj namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: bb734e6a-0f34-4b49-a994-9fbdda88f368 kind: Event lastTimestamp: "2026-07-08T17:04:22Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:55:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:22Z" name: error-404-isvc-1f879-predictor-5d845d546c-bwfkj.18c05ed16ce668e8 namespace: kserve-ci-e2e-test resourceVersion: "25442" uid: 1e310fdb-765c-4a79-b474-2095f77f913f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1f879-predictor-5d845d546c-bwfkj namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: bb734e6a-0f34-4b49-a994-9fbdda88f368 kind: Event lastTimestamp: "2026-07-08T17:04:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T17:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:19Z" name: error-404-isvc-1f879-predictor-5d845d546c-bwfkj.18c05f525004c644 namespace: kserve-ci-e2e-test resourceVersion: "25372" uid: 2e60059d-2bcf-4377-909c-53045c56e8b6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1f879-predictor-5d845d546c-bwfkj namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: bb734e6a-0f34-4b49-a994-9fbdda88f368 kind: Event lastTimestamp: "2026-07-08T17:04:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:19Z" name: error-404-isvc-1f879-predictor-5d845d546c-bwfkj.18c05f5250059546 namespace: kserve-ci-e2e-test resourceVersion: "25374" uid: 9c10bf81-1a49-4ef3-a2d2-75b240e14b4a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1f879-predictor-5d845d546c-bwfkj namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: bb734e6a-0f34-4b49-a994-9fbdda88f368 kind: Event lastTimestamp: "2026-07-08T17:04:22Z" message: 'Readiness probe failed: Get "https://10.133.0.27:8643/healthz": dial tcp 10.133.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:04:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:22Z" name: error-404-isvc-1f879-predictor-5d845d546c-bwfkj.18c05f52e145fc04 namespace: kserve-ci-e2e-test resourceVersion: "25441" uid: 02af318a-04cc-4bed-8389-d9e58effe18b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1f879-predictor-5d845d546c namespace: kserve-ci-e2e-test resourceVersion: "20719" uid: 4695fb11-308e-404d-834b-3709fcb04997 kind: Event lastTimestamp: "2026-07-08T16:55:04Z" message: 'Created pod: error-404-isvc-1f879-predictor-5d845d546c-bwfkj' metadata: creationTimestamp: "2026-07-08T16:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:04Z" name: error-404-isvc-1f879-predictor-5d845d546c.18c05ed1241e8d7b namespace: kserve-ci-e2e-test resourceVersion: "20730" uid: 40d838a5-9798-40ac-91e8-418ccde12d54 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1f879-predictor namespace: kserve-ci-e2e-test resourceVersion: "20715" uid: 79137059-c96c-408a-a2f1-1d15fc730308 kind: Event lastTimestamp: "2026-07-08T16:55:04Z" message: Scaled up replica set error-404-isvc-1f879-predictor-5d845d546c from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:04Z" name: error-404-isvc-1f879-predictor.18c05ed121a4dca4 namespace: kserve-ci-e2e-test resourceVersion: "20722" uid: f60b9b68-58e7-4a39-9ade-04badb1452dd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:55:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1f879-predictor namespace: kserve-ci-e2e-test resourceVersion: "20717" uid: b2e700bb-688c-4f0c-801f-ad141922dd43 kind: Event lastTimestamp: "2026-07-08T16:55:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:55:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:49Z" name: error-404-isvc-1f879-predictor.18c05ed4a0144fa2 namespace: kserve-ci-e2e-test resourceVersion: "21347" uid: b785a2ee-2f9d-41f7-b9c2-dfac10876da5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:55:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1f879-predictor namespace: kserve-ci-e2e-test resourceVersion: "20717" uid: b2e700bb-688c-4f0c-801f-ad141922dd43 kind: Event lastTimestamp: "2026-07-08T16:55:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:55:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:49Z" name: error-404-isvc-1f879-predictor.18c05ed4a0885b4f namespace: kserve-ci-e2e-test resourceVersion: "21348" uid: d5a2f9d0-130d-45e2-be5e-e92f9ed0a6cd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:55:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1f879-predictor namespace: kserve-ci-e2e-test resourceVersion: "20905" uid: b2e700bb-688c-4f0c-801f-ad141922dd43 kind: Event lastTimestamp: "2026-07-08T16:56:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:55:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:56:34Z" name: error-404-isvc-1f879-predictor.18c05ed81f4664b5 namespace: kserve-ci-e2e-test resourceVersion: "21786" uid: c334f225-beb3-4e1b-a83c-727e9d3f0536 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:55:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1f879-predictor namespace: kserve-ci-e2e-test resourceVersion: "20905" uid: b2e700bb-688c-4f0c-801f-ad141922dd43 kind: Event lastTimestamp: "2026-07-08T16:56:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:55:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:56:34Z" name: error-404-isvc-1f879-predictor.18c05ed81f9e6664 namespace: kserve-ci-e2e-test resourceVersion: "21787" uid: 94f206df-4008-4614-ba4a-a11bbbb74716 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:55:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1f879 namespace: kserve-ci-e2e-test resourceVersion: "20700" uid: 076ca635-7548-4557-bdc1-c691ccd575d8 kind: Event lastTimestamp: "2026-07-08T16:55:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:55:52Z" name: error-404-isvc-1f879.18c05ed121a7028e namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: 9625a122-32f2-4563-bfb2-07062ee4f7f0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1f879 namespace: kserve-ci-e2e-test resourceVersion: "20700" uid: 076ca635-7548-4557-bdc1-c691ccd575d8 kind: Event lastTimestamp: "2026-07-08T16:55:04Z" message: 'Failed to update status for InferenceService "error-404-isvc-1f879": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1f879": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:55:04Z" name: error-404-isvc-1f879.18c05ed1221130a7 namespace: kserve-ci-e2e-test resourceVersion: "20724" uid: 7d251d8d-ccb2-48c5-8502-beec91c147f5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1f879 namespace: kserve-ci-e2e-test resourceVersion: "20700" uid: 076ca635-7548-4557-bdc1-c691ccd575d8 kind: Event lastTimestamp: "2026-07-08T16:55:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1f879": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:55:04Z" name: error-404-isvc-1f879.18c05ed122138c4d namespace: kserve-ci-e2e-test resourceVersion: "20725" uid: fec311a8-60df-43ca-851e-6a146610a2ea reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1f879 namespace: kserve-ci-e2e-test resourceVersion: "21386" uid: 076ca635-7548-4557-bdc1-c691ccd575d8 kind: Event lastTimestamp: "2026-07-08T16:55:52Z" message: InferenceService [error-404-isvc-1f879] is Ready metadata: creationTimestamp: "2026-07-08T16:55:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:55:52Z" name: error-404-isvc-1f879.18c05edc2830d04b namespace: kserve-ci-e2e-test resourceVersion: "21387" uid: f8b340ea-778f-415f-8666-df4656c6fc19 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:54:34.294980Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3816e-predictor-65dc76b695-ckgw9 namespace: kserve-ci-e2e-test resourceVersion: "20280" uid: bfe92b54-88f7-44a6-aa95-a0d49fbf63d5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3816e-predictor-65dc76b695-ckgw9 to ip-10-0-138-233.ec2.internal metadata: creationTimestamp: "2026-07-08T16:54:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:54:34Z" name: error-404-isvc-3816e-predictor-65dc76b695-ckgw9.18c05eca0ab88d22 namespace: kserve-ci-e2e-test resourceVersion: "20284" uid: 7f419533-11a1-4910-a31c-f37af7da73e5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:34Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3816e-predictor-65dc76b695-ckgw9 namespace: kserve-ci-e2e-test resourceVersion: "20281" uid: bfe92b54-88f7-44a6-aa95-a0d49fbf63d5 kind: Event lastTimestamp: "2026-07-08T16:54:34Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-3816e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T16:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:34Z" name: error-404-isvc-3816e-predictor-65dc76b695-ckgw9.18c05eca1510b916 namespace: kserve-ci-e2e-test resourceVersion: "20297" uid: 4cfff673-862c-425d-9ee1-626937f47337 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:35Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3816e-predictor-65dc76b695-ckgw9 namespace: kserve-ci-e2e-test resourceVersion: "20290" uid: bfe92b54-88f7-44a6-aa95-a0d49fbf63d5 kind: Event lastTimestamp: "2026-07-08T16:54:35Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:54:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:54:35Z" name: error-404-isvc-3816e-predictor-65dc76b695-ckgw9.18c05eca480cdf44 namespace: kserve-ci-e2e-test resourceVersion: "20319" uid: 681e7945-098a-4953-89ee-060a362447ac reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3816e-predictor-65dc76b695-ckgw9 namespace: kserve-ci-e2e-test resourceVersion: "20281" uid: bfe92b54-88f7-44a6-aa95-a0d49fbf63d5 kind: Event lastTimestamp: "2026-07-08T16:54:35Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1707" already present on machine metadata: creationTimestamp: "2026-07-08T16:54:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:35Z" name: error-404-isvc-3816e-predictor-65dc76b695-ckgw9.18c05eca497b94f5 namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: e3ccfdec-a618-4d68-b2d4-9a937839eacc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3816e-predictor-65dc76b695-ckgw9 namespace: kserve-ci-e2e-test resourceVersion: "20281" uid: bfe92b54-88f7-44a6-aa95-a0d49fbf63d5 kind: Event lastTimestamp: "2026-07-08T16:54:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:54:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:35Z" name: error-404-isvc-3816e-predictor-65dc76b695-ckgw9.18c05eca4cb4a63c namespace: kserve-ci-e2e-test resourceVersion: "20326" uid: 03af3faf-fbb3-4314-9500-c49b81ce6b14 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3816e-predictor-65dc76b695-ckgw9 namespace: kserve-ci-e2e-test resourceVersion: "20281" uid: bfe92b54-88f7-44a6-aa95-a0d49fbf63d5 kind: Event lastTimestamp: "2026-07-08T16:54:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:54:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:35Z" name: error-404-isvc-3816e-predictor-65dc76b695-ckgw9.18c05eca4d4297fb namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: 7f9de95b-05f9-4547-8304-3f736f81fbd1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3816e-predictor-65dc76b695-ckgw9 namespace: kserve-ci-e2e-test resourceVersion: "20281" uid: bfe92b54-88f7-44a6-aa95-a0d49fbf63d5 kind: Event lastTimestamp: "2026-07-08T16:54:35Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:54:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:35Z" name: error-404-isvc-3816e-predictor-65dc76b695-ckgw9.18c05eca4d4c2d9f namespace: kserve-ci-e2e-test resourceVersion: "20328" uid: a2a2f5ff-830f-4628-a060-ddaa69bd4f1e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3816e-predictor-65dc76b695-ckgw9 namespace: kserve-ci-e2e-test resourceVersion: "20281" uid: bfe92b54-88f7-44a6-aa95-a0d49fbf63d5 kind: Event lastTimestamp: "2026-07-08T16:54:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:54:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:35Z" name: error-404-isvc-3816e-predictor-65dc76b695-ckgw9.18c05eca503b2ca8 namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: f37ea607-fde7-42fd-91f6-9df980e09425 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3816e-predictor-65dc76b695-ckgw9 namespace: kserve-ci-e2e-test resourceVersion: "20281" uid: bfe92b54-88f7-44a6-aa95-a0d49fbf63d5 kind: Event lastTimestamp: "2026-07-08T16:54:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:54:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:35Z" name: error-404-isvc-3816e-predictor-65dc76b695-ckgw9.18c05eca50b326ca namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 4ed01454-a316-4b1d-a546-848b3281570f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:54:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3816e-predictor-65dc76b695-ckgw9 namespace: kserve-ci-e2e-test resourceVersion: "20281" uid: bfe92b54-88f7-44a6-aa95-a0d49fbf63d5 kind: Event lastTimestamp: "2026-07-08T16:55:11Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:54:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:11Z" name: error-404-isvc-3816e-predictor-65dc76b695-ckgw9.18c05eca693e1533 namespace: kserve-ci-e2e-test resourceVersion: "20838" uid: 5a0c73ab-b02c-4b6a-81b0-a31df3f0e112 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3816e-predictor-65dc76b695-ckgw9 namespace: kserve-ci-e2e-test resourceVersion: "20281" uid: bfe92b54-88f7-44a6-aa95-a0d49fbf63d5 kind: Event lastTimestamp: "2026-07-08T16:55:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:44Z" name: error-404-isvc-3816e-predictor-65dc76b695-ckgw9.18c05eda60ac3b78 namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: 146fe2be-d5ec-499c-b5a3-6c7320f270a3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3816e-predictor-65dc76b695-ckgw9 namespace: kserve-ci-e2e-test resourceVersion: "20281" uid: bfe92b54-88f7-44a6-aa95-a0d49fbf63d5 kind: Event lastTimestamp: "2026-07-08T16:55:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:44Z" name: error-404-isvc-3816e-predictor-65dc76b695-ckgw9.18c05eda60aca9e6 namespace: kserve-ci-e2e-test resourceVersion: "21222" uid: b96e7cf3-f8f0-4152-a28e-72ced48a5eea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3816e-predictor-65dc76b695-ckgw9 namespace: kserve-ci-e2e-test resourceVersion: "20281" uid: bfe92b54-88f7-44a6-aa95-a0d49fbf63d5 kind: Event lastTimestamp: "2026-07-08T16:55:46Z" message: 'Readiness probe failed: Get "https://10.133.0.26:8643/healthz": dial tcp 10.133.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:55:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:46Z" name: error-404-isvc-3816e-predictor-65dc76b695-ckgw9.18c05edaf166b98f namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: dea78848-1c18-4e71-bb82-eb5242801110 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3816e-predictor-65dc76b695 namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: a8c3a7b2-1346-4da3-8045-21d1fea09712 kind: Event lastTimestamp: "2026-07-08T16:54:34Z" message: 'Created pod: error-404-isvc-3816e-predictor-65dc76b695-ckgw9' metadata: creationTimestamp: "2026-07-08T16:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:54:34Z" name: error-404-isvc-3816e-predictor-65dc76b695.18c05eca0a541038 namespace: kserve-ci-e2e-test resourceVersion: "20282" uid: e549879f-5ab5-491b-b94f-fbcf36c74bf0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3816e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20265" uid: 550a48e5-71d6-473f-a853-ab554c66b031 kind: Event lastTimestamp: "2026-07-08T16:54:34Z" message: Scaled up replica set error-404-isvc-3816e-predictor-65dc76b695 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:54:34Z" name: error-404-isvc-3816e-predictor.18c05eca0878a07f namespace: kserve-ci-e2e-test resourceVersion: "20271" uid: e1589250-1179-4df5-afb2-37434bc023a8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:54:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3816e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: 493edf48-5fa9-4579-889e-172495bcfe56 kind: Event lastTimestamp: "2026-07-08T16:55:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:54:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:19Z" name: error-404-isvc-3816e-predictor.18c05ecd871935ef namespace: kserve-ci-e2e-test resourceVersion: "20895" uid: a3482ca2-20be-4080-b7e7-4f8c99f5f277 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:54:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3816e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: 493edf48-5fa9-4579-889e-172495bcfe56 kind: Event lastTimestamp: "2026-07-08T16:55:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:54:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:19Z" name: error-404-isvc-3816e-predictor.18c05ecd878c80a6 namespace: kserve-ci-e2e-test resourceVersion: "20896" uid: fc01cf4d-7cb1-4f70-bbc5-e507076c70de reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3816e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20470" uid: 493edf48-5fa9-4579-889e-172495bcfe56 kind: Event lastTimestamp: "2026-07-08T16:55:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:55:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:34Z" name: error-404-isvc-3816e-predictor.18c05ed804144d2c namespace: kserve-ci-e2e-test resourceVersion: "21058" uid: e9e10885-7362-4a5b-a451-1ee10e38c7d3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3816e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20470" uid: 493edf48-5fa9-4579-889e-172495bcfe56 kind: Event lastTimestamp: "2026-07-08T16:55:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:55:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:34Z" name: error-404-isvc-3816e-predictor.18c05ed8048c6628 namespace: kserve-ci-e2e-test resourceVersion: "21060" uid: e58baa74-6b3a-499e-8811-16c8513d01a2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:54:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3816e namespace: kserve-ci-e2e-test resourceVersion: "20240" uid: bebebd0e-7822-495a-aa54-00b531b8d272 kind: Event lastTimestamp: "2026-07-08T16:55:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:55:22Z" name: error-404-isvc-3816e.18c05eca08a645e8 namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: a21e036a-b9e1-464b-86c7-7f39c81931d1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3816e namespace: kserve-ci-e2e-test resourceVersion: "20240" uid: bebebd0e-7822-495a-aa54-00b531b8d272 kind: Event lastTimestamp: "2026-07-08T16:54:34Z" message: 'Failed to update status for InferenceService "error-404-isvc-3816e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3816e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:54:34Z" name: error-404-isvc-3816e.18c05eca08febe76 namespace: kserve-ci-e2e-test resourceVersion: "20275" uid: 62daedaa-6cfc-48b8-9a8d-de780f900ece reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3816e namespace: kserve-ci-e2e-test resourceVersion: "20240" uid: bebebd0e-7822-495a-aa54-00b531b8d272 kind: Event lastTimestamp: "2026-07-08T16:54:34Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3816e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:54:34Z" name: error-404-isvc-3816e.18c05eca08ff01ac namespace: kserve-ci-e2e-test resourceVersion: "20277" uid: 4853db27-34f1-4f03-b712-d2fd03360394 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3816e namespace: kserve-ci-e2e-test resourceVersion: "20938" uid: bebebd0e-7822-495a-aa54-00b531b8d272 kind: Event lastTimestamp: "2026-07-08T16:55:21Z" message: InferenceService [error-404-isvc-3816e] is Ready metadata: creationTimestamp: "2026-07-08T16:55:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:55:21Z" name: error-404-isvc-3816e.18c05ed524b8e9b8 namespace: kserve-ci-e2e-test resourceVersion: "20939" uid: dc1af2bb-c72b-4e45-a90f-de203bf3c571 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T17:04:59.564661Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-73f94-predictor-696488f899-klrs9 namespace: kserve-ci-e2e-test resourceVersion: "25825" uid: 5aa1c3be-fd8e-44d5-8f68-d3e1fc37c399 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-73f94-predictor-696488f899-klrs9 to ip-10-0-138-233.ec2.internal metadata: creationTimestamp: "2026-07-08T17:04:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T17:04:59Z" name: error-404-isvc-73f94-predictor-696488f899-klrs9.18c05f5b9fb2ab94 namespace: kserve-ci-e2e-test resourceVersion: "25829" uid: 1eaea8a8-c293-4c05-845f-312c81a2288e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:59Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-73f94-predictor-696488f899-klrs9 namespace: kserve-ci-e2e-test resourceVersion: "25835" uid: 5aa1c3be-fd8e-44d5-8f68-d3e1fc37c399 kind: Event lastTimestamp: "2026-07-08T17:04:59Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T17:04:59Z" name: error-404-isvc-73f94-predictor-696488f899-klrs9.18c05f5bb931bc76 namespace: kserve-ci-e2e-test resourceVersion: "25846" uid: 8797cce7-858e-477f-bb8c-fbde62bcb325 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-73f94-predictor-696488f899-klrs9 namespace: kserve-ci-e2e-test resourceVersion: "25827" uid: 5aa1c3be-fd8e-44d5-8f68-d3e1fc37c399 kind: Event lastTimestamp: "2026-07-08T17:05:00Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1707" already present on machine metadata: creationTimestamp: "2026-07-08T17:05:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:00Z" name: error-404-isvc-73f94-predictor-696488f899-klrs9.18c05f5bbac4094f namespace: kserve-ci-e2e-test resourceVersion: "25848" uid: 9ab2e198-87fa-4c55-b88c-b07ca7a649f6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-73f94-predictor-696488f899-klrs9 namespace: kserve-ci-e2e-test resourceVersion: "25827" uid: 5aa1c3be-fd8e-44d5-8f68-d3e1fc37c399 kind: Event lastTimestamp: "2026-07-08T17:05:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T17:05:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:00Z" name: error-404-isvc-73f94-predictor-696488f899-klrs9.18c05f5bbd80dbe0 namespace: kserve-ci-e2e-test resourceVersion: "25850" uid: 5032928f-dc08-4b24-a2d4-85e5ca4e2dad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-73f94-predictor-696488f899-klrs9 namespace: kserve-ci-e2e-test resourceVersion: "25827" uid: 5aa1c3be-fd8e-44d5-8f68-d3e1fc37c399 kind: Event lastTimestamp: "2026-07-08T17:05:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T17:05:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:00Z" name: error-404-isvc-73f94-predictor-696488f899-klrs9.18c05f5bbdf65e0c namespace: kserve-ci-e2e-test resourceVersion: "25852" uid: 92cee775-9cc0-4a6f-8c12-c5b0826edf34 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-73f94-predictor-696488f899-klrs9 namespace: kserve-ci-e2e-test resourceVersion: "25827" uid: 5aa1c3be-fd8e-44d5-8f68-d3e1fc37c399 kind: Event lastTimestamp: "2026-07-08T17:05:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T17:05:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:00Z" name: error-404-isvc-73f94-predictor-696488f899-klrs9.18c05f5bbe019ae0 namespace: kserve-ci-e2e-test resourceVersion: "25853" uid: f9f58cec-d9f0-4548-8b26-28978c377119 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-73f94-predictor-696488f899-klrs9 namespace: kserve-ci-e2e-test resourceVersion: "25827" uid: 5aa1c3be-fd8e-44d5-8f68-d3e1fc37c399 kind: Event lastTimestamp: "2026-07-08T17:05:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T17:05:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:00Z" name: error-404-isvc-73f94-predictor-696488f899-klrs9.18c05f5bc1b0d426 namespace: kserve-ci-e2e-test resourceVersion: "25856" uid: f6a23ebb-93e9-421b-90b0-5152afc7a3d3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-73f94-predictor-696488f899-klrs9 namespace: kserve-ci-e2e-test resourceVersion: "25827" uid: 5aa1c3be-fd8e-44d5-8f68-d3e1fc37c399 kind: Event lastTimestamp: "2026-07-08T17:05:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:05:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:00Z" name: error-404-isvc-73f94-predictor-696488f899-klrs9.18c05f5bc222ab50 namespace: kserve-ci-e2e-test resourceVersion: "25857" uid: 9cd38076-c28b-4705-a8d3-f352076cecb5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T17:05:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-73f94-predictor-696488f899-klrs9 namespace: kserve-ci-e2e-test resourceVersion: "25827" uid: 5aa1c3be-fd8e-44d5-8f68-d3e1fc37c399 kind: Event lastTimestamp: "2026-07-08T17:05:37Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:05:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:37Z" name: error-404-isvc-73f94-predictor-696488f899-klrs9.18c05f5c0bda1063 namespace: kserve-ci-e2e-test resourceVersion: "26375" uid: 73123cc2-d381-4bfc-8db4-1cc08cc46d8c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:22:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-73f94-predictor-696488f899-klrs9 namespace: kserve-ci-e2e-test resourceVersion: "25827" uid: 5aa1c3be-fd8e-44d5-8f68-d3e1fc37c399 kind: Event lastTimestamp: "2026-07-08T17:22:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T17:22:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:22:18Z" name: error-404-isvc-73f94-predictor-696488f899-klrs9.18c0604da11484d4 namespace: kserve-ci-e2e-test resourceVersion: "33510" uid: 1412a2f9-fe01-40d2-9c98-69ca2c8a10d0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:22:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-73f94-predictor-696488f899-klrs9 namespace: kserve-ci-e2e-test resourceVersion: "25827" uid: 5aa1c3be-fd8e-44d5-8f68-d3e1fc37c399 kind: Event lastTimestamp: "2026-07-08T17:22:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:22:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:22:18Z" name: error-404-isvc-73f94-predictor-696488f899-klrs9.18c0604da1150e0b namespace: kserve-ci-e2e-test resourceVersion: "33511" uid: f09204d9-e872-46ac-9ab4-cd6e5a2e2ebd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:22:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-73f94-predictor-696488f899-klrs9 namespace: kserve-ci-e2e-test resourceVersion: "25827" uid: 5aa1c3be-fd8e-44d5-8f68-d3e1fc37c399 kind: Event lastTimestamp: "2026-07-08T17:22:22Z" message: 'Readiness probe failed: Get "https://10.133.0.30:8643/healthz": dial tcp 10.133.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:22:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:22:22Z" name: error-404-isvc-73f94-predictor-696488f899-klrs9.18c0604e6c8181cf namespace: kserve-ci-e2e-test resourceVersion: "33568" uid: c4f10ae2-21c0-4b90-be5f-2827a168c30f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-73f94-predictor-696488f899 namespace: kserve-ci-e2e-test resourceVersion: "25816" uid: b9b3349b-114b-4ede-9c5b-481d12bf358c kind: Event lastTimestamp: "2026-07-08T17:04:59Z" message: 'Created pod: error-404-isvc-73f94-predictor-696488f899-klrs9' metadata: creationTimestamp: "2026-07-08T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:04:59Z" name: error-404-isvc-73f94-predictor-696488f899.18c05f5b9f227c25 namespace: kserve-ci-e2e-test resourceVersion: "25826" uid: c85218e0-f3da-403b-9b4a-c9445d87ffae reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-73f94-predictor namespace: kserve-ci-e2e-test resourceVersion: "25813" uid: e1112bca-3843-48ec-b452-9e448f2926d4 kind: Event lastTimestamp: "2026-07-08T17:04:59Z" message: Scaled up replica set error-404-isvc-73f94-predictor-696488f899 from 0 to 1 metadata: creationTimestamp: "2026-07-08T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:04:59Z" name: error-404-isvc-73f94-predictor.18c05f5b9d596de3 namespace: kserve-ci-e2e-test resourceVersion: "25818" uid: 8cef358e-4d0a-4092-98d3-e34b7de9ae9f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:05:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-73f94-predictor namespace: kserve-ci-e2e-test resourceVersion: "25815" uid: 6af97b49-5d2e-4ae0-884b-0e90872493a9 kind: Event lastTimestamp: "2026-07-08T17:05:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:05:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:05:29Z" name: error-404-isvc-73f94-predictor.18c05f5f1bf634a1 namespace: kserve-ci-e2e-test resourceVersion: "26168" uid: d93816d7-f33f-4194-a37a-1c90a1f56071 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:05:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-73f94-predictor namespace: kserve-ci-e2e-test resourceVersion: "25815" uid: 6af97b49-5d2e-4ae0-884b-0e90872493a9 kind: Event lastTimestamp: "2026-07-08T17:05:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:05:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:05:29Z" name: error-404-isvc-73f94-predictor.18c05f5f1c65ed73 namespace: kserve-ci-e2e-test resourceVersion: "26169" uid: 73d56e89-1277-4522-8c51-63d238542644 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T17:05:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-73f94-predictor namespace: kserve-ci-e2e-test resourceVersion: "26012" uid: 6af97b49-5d2e-4ae0-884b-0e90872493a9 kind: Event lastTimestamp: "2026-07-08T17:06:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:06:14Z" name: error-404-isvc-73f94-predictor.18c05f661a08a109 namespace: kserve-ci-e2e-test resourceVersion: "26734" uid: 6968e0a3-2b88-4153-af95-510017e26d80 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T17:05:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-73f94-predictor namespace: kserve-ci-e2e-test resourceVersion: "26012" uid: 6af97b49-5d2e-4ae0-884b-0e90872493a9 kind: Event lastTimestamp: "2026-07-08T17:06:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:06:14Z" name: error-404-isvc-73f94-predictor.18c05f661a79c946 namespace: kserve-ci-e2e-test resourceVersion: "26735" uid: 791c7882-0905-4f64-8b72-2d2bccdc067b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T17:04:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-73f94 namespace: kserve-ci-e2e-test resourceVersion: "25796" uid: b359f92b-438f-44e0-a867-163c398deaa6 kind: Event lastTimestamp: "2026-07-08T17:05:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:05:47Z" name: error-404-isvc-73f94.18c05f5b9d80d1d2 namespace: kserve-ci-e2e-test resourceVersion: "26468" uid: 11bfb6ce-2404-4fca-95f2-e145b8e0b322 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-73f94 namespace: kserve-ci-e2e-test resourceVersion: "25796" uid: b359f92b-438f-44e0-a867-163c398deaa6 kind: Event lastTimestamp: "2026-07-08T17:04:59Z" message: 'Failed to update status for InferenceService "error-404-isvc-73f94": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-73f94": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:04:59Z" name: error-404-isvc-73f94.18c05f5b9e0054e2 namespace: kserve-ci-e2e-test resourceVersion: "25822" uid: 6ef52504-9958-44c6-b9a4-b3e7fccfb411 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-73f94 namespace: kserve-ci-e2e-test resourceVersion: "25796" uid: b359f92b-438f-44e0-a867-163c398deaa6 kind: Event lastTimestamp: "2026-07-08T17:04:59Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-73f94": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:04:59Z" name: error-404-isvc-73f94.18c05f5b9e009a06 namespace: kserve-ci-e2e-test resourceVersion: "25824" uid: c7089707-c609-4a0f-ad9a-d351de1f6ccf reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-73f94 namespace: kserve-ci-e2e-test resourceVersion: "26466" uid: b359f92b-438f-44e0-a867-163c398deaa6 kind: Event lastTimestamp: "2026-07-08T17:05:47Z" message: InferenceService [error-404-isvc-73f94] is Ready metadata: creationTimestamp: "2026-07-08T17:05:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:05:47Z" name: error-404-isvc-73f94.18c05f66c74506b6 namespace: kserve-ci-e2e-test resourceVersion: "26467" uid: 0a2ecb7a-2fdd-4a79-85f4-a22afcb443f7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T17:04:19.598646Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-78c4a-predictor-7c46b556f9-mhct7 namespace: kserve-ci-e2e-test resourceVersion: "25377" uid: ceed8830-50d6-4bc2-81c7-eade2692b180 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-78c4a-predictor-7c46b556f9-mhct7 to ip-10-0-138-233.ec2.internal metadata: creationTimestamp: "2026-07-08T17:04:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T17:04:19Z" name: error-404-isvc-78c4a-predictor-7c46b556f9-mhct7.18c05f525189f2e7 namespace: kserve-ci-e2e-test resourceVersion: "25381" uid: 2f32f671-dfde-4cb4-bcde-869754d86e64 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:20Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-78c4a-predictor-7c46b556f9-mhct7 namespace: kserve-ci-e2e-test resourceVersion: "25385" uid: ceed8830-50d6-4bc2-81c7-eade2692b180 kind: Event lastTimestamp: "2026-07-08T17:04:20Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T17:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T17:04:20Z" name: error-404-isvc-78c4a-predictor-7c46b556f9-mhct7.18c05f526b0cbb0b namespace: kserve-ci-e2e-test resourceVersion: "25397" uid: 162030bc-326b-4a70-a571-c0348bbaaf84 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78c4a-predictor-7c46b556f9-mhct7 namespace: kserve-ci-e2e-test resourceVersion: "25379" uid: ceed8830-50d6-4bc2-81c7-eade2692b180 kind: Event lastTimestamp: "2026-07-08T17:04:20Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1707" already present on machine metadata: creationTimestamp: "2026-07-08T17:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:20Z" name: error-404-isvc-78c4a-predictor-7c46b556f9-mhct7.18c05f526c6315cd namespace: kserve-ci-e2e-test resourceVersion: "25399" uid: 96f6ca05-1858-4f03-ae16-c05b34aaa8e9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78c4a-predictor-7c46b556f9-mhct7 namespace: kserve-ci-e2e-test resourceVersion: "25379" uid: ceed8830-50d6-4bc2-81c7-eade2692b180 kind: Event lastTimestamp: "2026-07-08T17:04:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T17:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:20Z" name: error-404-isvc-78c4a-predictor-7c46b556f9-mhct7.18c05f526ef9e860 namespace: kserve-ci-e2e-test resourceVersion: "25400" uid: 66c04f06-bad3-4935-9a5f-19add9ee8b4c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78c4a-predictor-7c46b556f9-mhct7 namespace: kserve-ci-e2e-test resourceVersion: "25379" uid: ceed8830-50d6-4bc2-81c7-eade2692b180 kind: Event lastTimestamp: "2026-07-08T17:04:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T17:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:20Z" name: error-404-isvc-78c4a-predictor-7c46b556f9-mhct7.18c05f526f6f7cb7 namespace: kserve-ci-e2e-test resourceVersion: "25402" uid: 31899ef3-2d05-461c-8847-1fc4a7b4cecf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78c4a-predictor-7c46b556f9-mhct7 namespace: kserve-ci-e2e-test resourceVersion: "25379" uid: ceed8830-50d6-4bc2-81c7-eade2692b180 kind: Event lastTimestamp: "2026-07-08T17:04:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T17:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:20Z" name: error-404-isvc-78c4a-predictor-7c46b556f9-mhct7.18c05f526f7bbf44 namespace: kserve-ci-e2e-test resourceVersion: "25403" uid: 63fa4fbd-ab54-4631-a84e-b5699e4c7786 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78c4a-predictor-7c46b556f9-mhct7 namespace: kserve-ci-e2e-test resourceVersion: "25379" uid: ceed8830-50d6-4bc2-81c7-eade2692b180 kind: Event lastTimestamp: "2026-07-08T17:04:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T17:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:20Z" name: error-404-isvc-78c4a-predictor-7c46b556f9-mhct7.18c05f52740570a9 namespace: kserve-ci-e2e-test resourceVersion: "25404" uid: 9be1937d-8b25-42ba-ae3a-1db3cdbe6149 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78c4a-predictor-7c46b556f9-mhct7 namespace: kserve-ci-e2e-test resourceVersion: "25379" uid: ceed8830-50d6-4bc2-81c7-eade2692b180 kind: Event lastTimestamp: "2026-07-08T17:04:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:20Z" name: error-404-isvc-78c4a-predictor-7c46b556f9-mhct7.18c05f527476885e namespace: kserve-ci-e2e-test resourceVersion: "25405" uid: 3c16db48-2b12-4941-a2db-6b9bbbffbdb8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T17:04:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78c4a-predictor-7c46b556f9-mhct7 namespace: kserve-ci-e2e-test resourceVersion: "25379" uid: ceed8830-50d6-4bc2-81c7-eade2692b180 kind: Event lastTimestamp: "2026-07-08T17:04:56Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:04:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:56Z" name: error-404-isvc-78c4a-predictor-7c46b556f9-mhct7.18c05f52759d127c namespace: kserve-ci-e2e-test resourceVersion: "25704" uid: b3494ac7-e5e2-440b-8f98-0989167da5e4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78c4a-predictor-7c46b556f9-mhct7 namespace: kserve-ci-e2e-test resourceVersion: "25379" uid: ceed8830-50d6-4bc2-81c7-eade2692b180 kind: Event lastTimestamp: "2026-07-08T17:05:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T17:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:29Z" name: error-404-isvc-78c4a-predictor-7c46b556f9-mhct7.18c05f62aa192a33 namespace: kserve-ci-e2e-test resourceVersion: "26235" uid: ece404cc-386c-4838-b52c-ef66c7e86088 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78c4a-predictor-7c46b556f9-mhct7 namespace: kserve-ci-e2e-test resourceVersion: "25379" uid: ceed8830-50d6-4bc2-81c7-eade2692b180 kind: Event lastTimestamp: "2026-07-08T17:05:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:29Z" name: error-404-isvc-78c4a-predictor-7c46b556f9-mhct7.18c05f62aa19c445 namespace: kserve-ci-e2e-test resourceVersion: "26237" uid: adac2ddd-c628-41b9-89d7-d4b4e2e7832f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78c4a-predictor-7c46b556f9-mhct7 namespace: kserve-ci-e2e-test resourceVersion: "25379" uid: ceed8830-50d6-4bc2-81c7-eade2692b180 kind: Event lastTimestamp: "2026-07-08T17:05:31Z" message: 'Readiness probe failed: Get "https://10.133.0.29:8643/healthz": dial tcp 10.133.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:05:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:31Z" name: error-404-isvc-78c4a-predictor-7c46b556f9-mhct7.18c05f62fdc43d6d namespace: kserve-ci-e2e-test resourceVersion: "26316" uid: 6ba21e44-40e9-49ba-a380-727e06afc814 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-78c4a-predictor-7c46b556f9 namespace: kserve-ci-e2e-test resourceVersion: "25364" uid: 51a26bba-c109-4cd7-802a-62c2966ce9d9 kind: Event lastTimestamp: "2026-07-08T17:04:19Z" message: 'Created pod: error-404-isvc-78c4a-predictor-7c46b556f9-mhct7' metadata: creationTimestamp: "2026-07-08T17:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:04:19Z" name: error-404-isvc-78c4a-predictor-7c46b556f9.18c05f5251202ce7 namespace: kserve-ci-e2e-test resourceVersion: "25378" uid: 41b7db2f-4414-400b-938f-0b0928181186 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-78c4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25359" uid: 2dc6cf74-32c7-4a45-b92b-91873689da08 kind: Event lastTimestamp: "2026-07-08T17:04:19Z" message: Scaled up replica set error-404-isvc-78c4a-predictor-7c46b556f9 from 0 to 1 metadata: creationTimestamp: "2026-07-08T17:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:04:19Z" name: error-404-isvc-78c4a-predictor.18c05f524f6056ad namespace: kserve-ci-e2e-test resourceVersion: "25368" uid: 64c39fd7-bc81-43d9-8496-f5d290c57e5e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T17:04:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-78c4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25363" uid: a8abf96a-e007-4301-8d8e-96c3dde576e3 kind: Event lastTimestamp: "2026-07-08T17:05:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:04:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:05:19Z" name: error-404-isvc-78c4a-predictor.18c05f55cdedd575 namespace: kserve-ci-e2e-test resourceVersion: "26076" uid: a876151e-5dc0-4c5b-8342-506164f85364 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T17:04:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-78c4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25363" uid: a8abf96a-e007-4301-8d8e-96c3dde576e3 kind: Event lastTimestamp: "2026-07-08T17:05:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:04:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:05:19Z" name: error-404-isvc-78c4a-predictor.18c05f55ce59be01 namespace: kserve-ci-e2e-test resourceVersion: "26077" uid: 5bd46229-18d6-4f2d-ae95-bb1c61e0200e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T17:04:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-78c4a namespace: kserve-ci-e2e-test resourceVersion: "25336" uid: b8a02bd5-c72e-4a6f-9b8f-502780e0922a kind: Event lastTimestamp: "2026-07-08T17:05:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T17:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:05:06Z" name: error-404-isvc-78c4a.18c05f524f6a69f2 namespace: kserve-ci-e2e-test resourceVersion: "25934" uid: 454ce1be-b3a9-4da6-94d0-2bccc6674330 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-78c4a namespace: kserve-ci-e2e-test resourceVersion: "25336" uid: b8a02bd5-c72e-4a6f-9b8f-502780e0922a kind: Event lastTimestamp: "2026-07-08T17:04:19Z" message: 'Failed to update status for InferenceService "error-404-isvc-78c4a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-78c4a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T17:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:04:19Z" name: error-404-isvc-78c4a.18c05f525004dac8 namespace: kserve-ci-e2e-test resourceVersion: "25371" uid: 5168c62f-bf8c-4912-850a-4199b81d2716 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-78c4a namespace: kserve-ci-e2e-test resourceVersion: "25336" uid: b8a02bd5-c72e-4a6f-9b8f-502780e0922a kind: Event lastTimestamp: "2026-07-08T17:04:19Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-78c4a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T17:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:04:19Z" name: error-404-isvc-78c4a.18c05f5250051484 namespace: kserve-ci-e2e-test resourceVersion: "25373" uid: c2bac76c-ef10-48d7-933a-25b958a5310b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-78c4a namespace: kserve-ci-e2e-test resourceVersion: "25932" uid: b8a02bd5-c72e-4a6f-9b8f-502780e0922a kind: Event lastTimestamp: "2026-07-08T17:05:06Z" message: InferenceService [error-404-isvc-78c4a] is Ready metadata: creationTimestamp: "2026-07-08T17:05:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:05:06Z" name: error-404-isvc-78c4a.18c05f5d30f843a2 namespace: kserve-ci-e2e-test resourceVersion: "25933" uid: b4167f3f-37bb-4531-9f3f-f2d01f86e4ba reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:53:54.536984Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8befa-predictor-5957789c65-2q4tr namespace: kserve-ci-e2e-test resourceVersion: "19828" uid: 2979b4f3-c547-441d-9927-921676de4a1d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8befa-predictor-5957789c65-2q4tr to ip-10-0-138-233.ec2.internal metadata: creationTimestamp: "2026-07-08T16:53:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:53:54Z" name: error-404-isvc-8befa-predictor-5957789c65-2q4tr.18c05ec0c8f5becb namespace: kserve-ci-e2e-test resourceVersion: "19832" uid: 3adb8803-647b-4fc5-8fb5-530d44df60a6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:54Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8befa-predictor-5957789c65-2q4tr namespace: kserve-ci-e2e-test resourceVersion: "19829" uid: 2979b4f3-c547-441d-9927-921676de4a1d kind: Event lastTimestamp: "2026-07-08T16:53:54Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-8befa-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:54Z" name: error-404-isvc-8befa-predictor-5957789c65-2q4tr.18c05ec0d49647d4 namespace: kserve-ci-e2e-test resourceVersion: "19845" uid: 033cbbc8-3cad-4bd5-8eb1-088962d70a96 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:55Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8befa-predictor-5957789c65-2q4tr namespace: kserve-ci-e2e-test resourceVersion: "19837" uid: 2979b4f3-c547-441d-9927-921676de4a1d kind: Event lastTimestamp: "2026-07-08T16:53:55Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:53:55Z" name: error-404-isvc-8befa-predictor-5957789c65-2q4tr.18c05ec106a681e5 namespace: kserve-ci-e2e-test resourceVersion: "19869" uid: 9d870911-d348-4cfe-b0f8-a44a15ceec85 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8befa-predictor-5957789c65-2q4tr namespace: kserve-ci-e2e-test resourceVersion: "19829" uid: 2979b4f3-c547-441d-9927-921676de4a1d kind: Event lastTimestamp: "2026-07-08T16:53:55Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1707" already present on machine metadata: creationTimestamp: "2026-07-08T16:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:55Z" name: error-404-isvc-8befa-predictor-5957789c65-2q4tr.18c05ec10810c3dc namespace: kserve-ci-e2e-test resourceVersion: "19871" uid: 186242c0-3213-4aab-876a-2031bb6540cc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8befa-predictor-5957789c65-2q4tr namespace: kserve-ci-e2e-test resourceVersion: "19829" uid: 2979b4f3-c547-441d-9927-921676de4a1d kind: Event lastTimestamp: "2026-07-08T16:53:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:55Z" name: error-404-isvc-8befa-predictor-5957789c65-2q4tr.18c05ec10b2f91f8 namespace: kserve-ci-e2e-test resourceVersion: "19872" uid: 59582561-3f06-4141-997f-fd246e090583 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8befa-predictor-5957789c65-2q4tr namespace: kserve-ci-e2e-test resourceVersion: "19829" uid: 2979b4f3-c547-441d-9927-921676de4a1d kind: Event lastTimestamp: "2026-07-08T16:53:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:55Z" name: error-404-isvc-8befa-predictor-5957789c65-2q4tr.18c05ec10ba0e0d3 namespace: kserve-ci-e2e-test resourceVersion: "19873" uid: d7f13e2e-4b34-45f6-abf3-fb5bda2ac1d9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8befa-predictor-5957789c65-2q4tr namespace: kserve-ci-e2e-test resourceVersion: "19829" uid: 2979b4f3-c547-441d-9927-921676de4a1d kind: Event lastTimestamp: "2026-07-08T16:53:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:55Z" name: error-404-isvc-8befa-predictor-5957789c65-2q4tr.18c05ec10bad99ee namespace: kserve-ci-e2e-test resourceVersion: "19874" uid: 5a098c8c-aa98-4628-930d-0efea8c53849 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8befa-predictor-5957789c65-2q4tr namespace: kserve-ci-e2e-test resourceVersion: "19829" uid: 2979b4f3-c547-441d-9927-921676de4a1d kind: Event lastTimestamp: "2026-07-08T16:53:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:55Z" name: error-404-isvc-8befa-predictor-5957789c65-2q4tr.18c05ec10f03d725 namespace: kserve-ci-e2e-test resourceVersion: "19876" uid: b946aab3-f015-4e37-9275-e6952301ba2e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8befa-predictor-5957789c65-2q4tr namespace: kserve-ci-e2e-test resourceVersion: "19829" uid: 2979b4f3-c547-441d-9927-921676de4a1d kind: Event lastTimestamp: "2026-07-08T16:53:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:55Z" name: error-404-isvc-8befa-predictor-5957789c65-2q4tr.18c05ec10f7401fd namespace: kserve-ci-e2e-test resourceVersion: "19877" uid: 56cf7ac8-5a92-47e7-96d4-8afa75582c53 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:53:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8befa-predictor-5957789c65-2q4tr namespace: kserve-ci-e2e-test resourceVersion: "19829" uid: 2979b4f3-c547-441d-9927-921676de4a1d kind: Event lastTimestamp: "2026-07-08T16:54:33Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:33Z" name: error-404-isvc-8befa-predictor-5957789c65-2q4tr.18c05ec186faab44 namespace: kserve-ci-e2e-test resourceVersion: "20183" uid: f3045bec-9205-4da9-b8eb-a551397a7d58 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8befa-predictor-5957789c65-2q4tr namespace: kserve-ci-e2e-test resourceVersion: "19829" uid: 2979b4f3-c547-441d-9927-921676de4a1d kind: Event lastTimestamp: "2026-07-08T16:55:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:04Z" name: error-404-isvc-8befa-predictor-5957789c65-2q4tr.18c05ed11cef760b namespace: kserve-ci-e2e-test resourceVersion: "20701" uid: 4312f09a-70a9-4a10-a953-6d2c6d1bd85c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8befa-predictor-5957789c65-2q4tr namespace: kserve-ci-e2e-test resourceVersion: "19829" uid: 2979b4f3-c547-441d-9927-921676de4a1d kind: Event lastTimestamp: "2026-07-08T16:55:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:04Z" name: error-404-isvc-8befa-predictor-5957789c65-2q4tr.18c05ed11cefe4d7 namespace: kserve-ci-e2e-test resourceVersion: "20704" uid: ae76bd86-3fbb-4aa5-8280-a8887652e0de reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8befa-predictor-5957789c65 namespace: kserve-ci-e2e-test resourceVersion: "19821" uid: 9066e961-26d5-4bdd-ae37-27b9c1d62111 kind: Event lastTimestamp: "2026-07-08T16:53:54Z" message: 'Created pod: error-404-isvc-8befa-predictor-5957789c65-2q4tr' metadata: creationTimestamp: "2026-07-08T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:53:54Z" name: error-404-isvc-8befa-predictor-5957789c65.18c05ec0c8791a9d namespace: kserve-ci-e2e-test resourceVersion: "19830" uid: 4db2cb83-0b41-416b-8d68-67f549e1682c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8befa-predictor namespace: kserve-ci-e2e-test resourceVersion: "19818" uid: 01998a36-140d-4794-826a-a4b566bc9ace kind: Event lastTimestamp: "2026-07-08T16:53:54Z" message: Scaled up replica set error-404-isvc-8befa-predictor-5957789c65 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:53:54Z" name: error-404-isvc-8befa-predictor.18c05ec0c68499d8 namespace: kserve-ci-e2e-test resourceVersion: "19822" uid: 40a75298-a7bb-47cb-a85c-206695f93079 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8befa-predictor namespace: kserve-ci-e2e-test resourceVersion: "19820" uid: 2b265082-5ab2-414b-8d69-0f0460a4d92e kind: Event lastTimestamp: "2026-07-08T16:54:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:54:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:54:09Z" name: error-404-isvc-8befa-predictor.18c05ec4455f7eae namespace: kserve-ci-e2e-test resourceVersion: "20002" uid: cbb5d9fe-7fd4-461d-bf82-43b965c74ab6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8befa-predictor namespace: kserve-ci-e2e-test resourceVersion: "19820" uid: 2b265082-5ab2-414b-8d69-0f0460a4d92e kind: Event lastTimestamp: "2026-07-08T16:54:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:54:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:54:09Z" name: error-404-isvc-8befa-predictor.18c05ec445def087 namespace: kserve-ci-e2e-test resourceVersion: "20004" uid: c58ab55d-74ef-4250-a032-069899a4eac3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:54:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8befa-predictor namespace: kserve-ci-e2e-test resourceVersion: "20003" uid: 2b265082-5ab2-414b-8d69-0f0460a4d92e kind: Event lastTimestamp: "2026-07-08T16:54:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:54:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:54:54Z" name: error-404-isvc-8befa-predictor.18c05ec7c4d97141 namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: 95ca14fc-9557-418c-a0cc-4b9403b96973 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:54:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8befa-predictor namespace: kserve-ci-e2e-test resourceVersion: "20003" uid: 2b265082-5ab2-414b-8d69-0f0460a4d92e kind: Event lastTimestamp: "2026-07-08T16:54:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:54:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:54:54Z" name: error-404-isvc-8befa-predictor.18c05ec7c552641a namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: 3a0a29cd-51b6-49ef-82bc-16aa98b504f0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:53:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8befa namespace: kserve-ci-e2e-test resourceVersion: "19803" uid: a22d5675-d7f7-4cec-b3b9-c14e75bc8327 kind: Event lastTimestamp: "2026-07-08T16:54:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:54:43Z" name: error-404-isvc-8befa.18c05ec0c68e979e namespace: kserve-ci-e2e-test resourceVersion: "20428" uid: 10456dd7-4599-43c0-87ad-6e2029ee97f5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8befa namespace: kserve-ci-e2e-test resourceVersion: "20426" uid: a22d5675-d7f7-4cec-b3b9-c14e75bc8327 kind: Event lastTimestamp: "2026-07-08T16:54:43Z" message: InferenceService [error-404-isvc-8befa] is Ready metadata: creationTimestamp: "2026-07-08T16:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:54:43Z" name: error-404-isvc-8befa.18c05ecc42a83372 namespace: kserve-ci-e2e-test resourceVersion: "20427" uid: 91a1f4f6-5424-4341-b3d1-4713fd992dd5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:44:39.508945Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8ceba-predictor-84f96f9559-tdxw6 namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: 1c3c40fc-3656-4905-a966-b6dac3528ffb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8ceba-predictor-84f96f9559-tdxw6 to ip-10-0-138-233.ec2.internal metadata: creationTimestamp: "2026-07-08T16:44:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:44:39Z" name: error-404-isvc-8ceba-predictor-84f96f9559-tdxw6.18c05e3f8eb5e012 namespace: kserve-ci-e2e-test resourceVersion: "15442" uid: 3e245aab-df1d-4460-b5f9-b55bbc62817e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:39Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8ceba-predictor-84f96f9559-tdxw6 namespace: kserve-ci-e2e-test resourceVersion: "15445" uid: 1c3c40fc-3656-4905-a966-b6dac3528ffb kind: Event lastTimestamp: "2026-07-08T16:44:39Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:44:39Z" name: error-404-isvc-8ceba-predictor-84f96f9559-tdxw6.18c05e3fa7ee6a47 namespace: kserve-ci-e2e-test resourceVersion: "15466" uid: af70b582-82d7-4123-bed8-2c29a0560745 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8ceba-predictor-84f96f9559-tdxw6 namespace: kserve-ci-e2e-test resourceVersion: "15440" uid: 1c3c40fc-3656-4905-a966-b6dac3528ffb kind: Event lastTimestamp: "2026-07-08T16:44:39Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1707" already present on machine metadata: creationTimestamp: "2026-07-08T16:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:39Z" name: error-404-isvc-8ceba-predictor-84f96f9559-tdxw6.18c05e3fa94d939d namespace: kserve-ci-e2e-test resourceVersion: "15468" uid: d64ebe74-1d0d-47ef-ab75-d3c6b1c0ed43 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8ceba-predictor-84f96f9559-tdxw6 namespace: kserve-ci-e2e-test resourceVersion: "15440" uid: 1c3c40fc-3656-4905-a966-b6dac3528ffb kind: Event lastTimestamp: "2026-07-08T16:44:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:40Z" name: error-404-isvc-8ceba-predictor-84f96f9559-tdxw6.18c05e3fabf34ae0 namespace: kserve-ci-e2e-test resourceVersion: "15471" uid: ee797e34-67a5-494a-9d8d-50f9de72650a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8ceba-predictor-84f96f9559-tdxw6 namespace: kserve-ci-e2e-test resourceVersion: "15440" uid: 1c3c40fc-3656-4905-a966-b6dac3528ffb kind: Event lastTimestamp: "2026-07-08T16:44:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:40Z" name: error-404-isvc-8ceba-predictor-84f96f9559-tdxw6.18c05e3fac595dc1 namespace: kserve-ci-e2e-test resourceVersion: "15472" uid: c7786c27-630a-409e-9557-dcba6280c755 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8ceba-predictor-84f96f9559-tdxw6 namespace: kserve-ci-e2e-test resourceVersion: "15440" uid: 1c3c40fc-3656-4905-a966-b6dac3528ffb kind: Event lastTimestamp: "2026-07-08T16:44:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:40Z" name: error-404-isvc-8ceba-predictor-84f96f9559-tdxw6.18c05e3fac6618d0 namespace: kserve-ci-e2e-test resourceVersion: "15473" uid: 2b7ccb33-8e0e-4387-bfb4-b341b9add8a4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8ceba-predictor-84f96f9559-tdxw6 namespace: kserve-ci-e2e-test resourceVersion: "15440" uid: 1c3c40fc-3656-4905-a966-b6dac3528ffb kind: Event lastTimestamp: "2026-07-08T16:44:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:40Z" name: error-404-isvc-8ceba-predictor-84f96f9559-tdxw6.18c05e3fafdd2453 namespace: kserve-ci-e2e-test resourceVersion: "15474" uid: ea0e2ca1-197b-4291-a52a-809a49668b2c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8ceba-predictor-84f96f9559-tdxw6 namespace: kserve-ci-e2e-test resourceVersion: "15440" uid: 1c3c40fc-3656-4905-a966-b6dac3528ffb kind: Event lastTimestamp: "2026-07-08T16:44:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:40Z" name: error-404-isvc-8ceba-predictor-84f96f9559-tdxw6.18c05e3fb0401399 namespace: kserve-ci-e2e-test resourceVersion: "15475" uid: e80eb42c-c038-45c3-9989-d89f93433969 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-08T16:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8ceba-predictor-84f96f9559-tdxw6 namespace: kserve-ci-e2e-test resourceVersion: "15440" uid: 1c3c40fc-3656-4905-a966-b6dac3528ffb kind: Event lastTimestamp: "2026-07-08T16:53:56Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:56Z" name: error-404-isvc-8ceba-predictor-84f96f9559-tdxw6.18c05e3fca5d1ae3 namespace: kserve-ci-e2e-test resourceVersion: "19884" uid: 0829792d-adee-45e0-bb42-1516c4717331 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8ceba-predictor-84f96f9559-tdxw6 namespace: kserve-ci-e2e-test resourceVersion: "15440" uid: 1c3c40fc-3656-4905-a966-b6dac3528ffb kind: Event lastTimestamp: "2026-07-08T16:53:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:54Z" name: error-404-isvc-8ceba-predictor-84f96f9559-tdxw6.18c05ec0c2ca8c26 namespace: kserve-ci-e2e-test resourceVersion: "19804" uid: a62d4184-a817-4610-93c3-b79180e179f3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8ceba-predictor-84f96f9559-tdxw6 namespace: kserve-ci-e2e-test resourceVersion: "15440" uid: 1c3c40fc-3656-4905-a966-b6dac3528ffb kind: Event lastTimestamp: "2026-07-08T16:53:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:54Z" name: error-404-isvc-8ceba-predictor-84f96f9559-tdxw6.18c05ec0c2cae91a namespace: kserve-ci-e2e-test resourceVersion: "19806" uid: 30d34819-fe07-4159-ad3a-49df8ea6f539 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8ceba-predictor-84f96f9559-tdxw6 namespace: kserve-ci-e2e-test resourceVersion: "15440" uid: 1c3c40fc-3656-4905-a966-b6dac3528ffb kind: Event lastTimestamp: "2026-07-08T16:53:56Z" message: 'Readiness probe failed: Get "https://10.133.0.23:8643/healthz": dial tcp 10.133.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:56Z" name: error-404-isvc-8ceba-predictor-84f96f9559-tdxw6.18c05ec13eba2866 namespace: kserve-ci-e2e-test resourceVersion: "19883" uid: ab14bb50-380c-4f58-83f0-4be696560a51 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8ceba-predictor-84f96f9559 namespace: kserve-ci-e2e-test resourceVersion: "15423" uid: 1c3c32a2-484f-4bb3-8330-25b76fc3e3eb kind: Event lastTimestamp: "2026-07-08T16:44:39Z" message: 'Created pod: error-404-isvc-8ceba-predictor-84f96f9559-tdxw6' metadata: creationTimestamp: "2026-07-08T16:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:44:39Z" name: error-404-isvc-8ceba-predictor-84f96f9559.18c05e3f8e40699c namespace: kserve-ci-e2e-test resourceVersion: "15439" uid: 62f91490-ec7d-4cb1-a75e-29f190744e7a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8ceba-predictor namespace: kserve-ci-e2e-test resourceVersion: "15416" uid: e3d6980f-8a93-4e2f-aa7f-3b24142712df kind: Event lastTimestamp: "2026-07-08T16:44:39Z" message: Scaled up replica set error-404-isvc-8ceba-predictor-84f96f9559 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:44:39Z" name: error-404-isvc-8ceba-predictor.18c05e3f8c47fc4d namespace: kserve-ci-e2e-test resourceVersion: "15428" uid: 54f0253a-bb35-403c-a404-218361c38c3f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:44:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8ceba-predictor namespace: kserve-ci-e2e-test resourceVersion: "15422" uid: ed6ba761-f688-499f-b812-0be8d1ef0bef kind: Event lastTimestamp: "2026-07-08T16:45:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:44:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:45:09Z" name: error-404-isvc-8ceba-predictor.18c05e430ad02829 namespace: kserve-ci-e2e-test resourceVersion: "15782" uid: 1c5c1274-03be-46d5-89df-8639ff93ba57 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:44:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8ceba-predictor namespace: kserve-ci-e2e-test resourceVersion: "15422" uid: ed6ba761-f688-499f-b812-0be8d1ef0bef kind: Event lastTimestamp: "2026-07-08T16:45:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:44:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:45:09Z" name: error-404-isvc-8ceba-predictor.18c05e430b4397aa namespace: kserve-ci-e2e-test resourceVersion: "15783" uid: e05aa8d1-7757-4de2-9a38-0e994c921e6c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:45:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8ceba-predictor namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: ed6ba761-f688-499f-b812-0be8d1ef0bef kind: Event lastTimestamp: "2026-07-08T16:45:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:45:39Z" name: error-404-isvc-8ceba-predictor.18c05e4a08dfc750 namespace: kserve-ci-e2e-test resourceVersion: "16241" uid: 8d9e8f6a-1a94-4dfd-8ed7-5da07acdff11 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:45:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8ceba-predictor namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: ed6ba761-f688-499f-b812-0be8d1ef0bef kind: Event lastTimestamp: "2026-07-08T16:45:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:45:39Z" name: error-404-isvc-8ceba-predictor.18c05e4a095452af namespace: kserve-ci-e2e-test resourceVersion: "16242" uid: 01a5b688-4012-4f6c-a9b4-74dc47f2a574 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:44:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8ceba namespace: kserve-ci-e2e-test resourceVersion: "15396" uid: bbc00606-7436-4374-8660-0c7fe6dd693b kind: Event lastTimestamp: "2026-07-08T16:45:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:45:26Z" name: error-404-isvc-8ceba.18c05e3f8c494c00 namespace: kserve-ci-e2e-test resourceVersion: "16105" uid: b570890c-c717-4052-9f1e-8aeb271c9394 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8ceba namespace: kserve-ci-e2e-test resourceVersion: "15396" uid: bbc00606-7436-4374-8660-0c7fe6dd693b kind: Event lastTimestamp: "2026-07-08T16:44:39Z" message: 'Failed to update status for InferenceService "error-404-isvc-8ceba": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8ceba": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:44:39Z" name: error-404-isvc-8ceba.18c05e3f8ce4d683 namespace: kserve-ci-e2e-test resourceVersion: "15432" uid: ce7706c0-c90c-449f-9329-05256e9214ff reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8ceba namespace: kserve-ci-e2e-test resourceVersion: "15396" uid: bbc00606-7436-4374-8660-0c7fe6dd693b kind: Event lastTimestamp: "2026-07-08T16:44:39Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8ceba": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:44:39Z" name: error-404-isvc-8ceba.18c05e3f8ce541af namespace: kserve-ci-e2e-test resourceVersion: "15433" uid: 233b5d53-4a91-4b2c-9912-a46f30fa4119 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8ceba namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: bbc00606-7436-4374-8660-0c7fe6dd693b kind: Event lastTimestamp: "2026-07-08T16:45:26Z" message: InferenceService [error-404-isvc-8ceba] is Ready metadata: creationTimestamp: "2026-07-08T16:45:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:45:26Z" name: error-404-isvc-8ceba.18c05e4a86196a7f namespace: kserve-ci-e2e-test resourceVersion: "16104" uid: 54128837-2d71-4e07-8f3e-c808f7439a67 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T17:05:29.938418Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-91fdc-predictor-5d77898ff9-wkvdq namespace: kserve-ci-e2e-test resourceVersion: "26267" uid: 6a5ec9e4-4350-4ed0-beca-d56c833516e7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-91fdc-predictor-5d77898ff9-wkvdq to ip-10-0-138-233.ec2.internal metadata: creationTimestamp: "2026-07-08T17:05:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T17:05:29Z" name: error-404-isvc-91fdc-predictor-5d77898ff9-wkvdq.18c05f62b21d76fb namespace: kserve-ci-e2e-test resourceVersion: "26272" uid: b26016d5-f220-434a-8b7d-0535c5b65495 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:30Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-91fdc-predictor-5d77898ff9-wkvdq namespace: kserve-ci-e2e-test resourceVersion: "26277" uid: 6a5ec9e4-4350-4ed0-beca-d56c833516e7 kind: Event lastTimestamp: "2026-07-08T17:05:30Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T17:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T17:05:30Z" name: error-404-isvc-91fdc-predictor-5d77898ff9-wkvdq.18c05f62cbbb41fe namespace: kserve-ci-e2e-test resourceVersion: "26300" uid: 9f7a7f9d-b912-4249-9c6a-1949ab499617 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-91fdc-predictor-5d77898ff9-wkvdq namespace: kserve-ci-e2e-test resourceVersion: "26269" uid: 6a5ec9e4-4350-4ed0-beca-d56c833516e7 kind: Event lastTimestamp: "2026-07-08T17:05:30Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1707" already present on machine metadata: creationTimestamp: "2026-07-08T17:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:30Z" name: error-404-isvc-91fdc-predictor-5d77898ff9-wkvdq.18c05f62cd2336b1 namespace: kserve-ci-e2e-test resourceVersion: "26302" uid: fbcf3da9-933f-445b-87a9-fab45c6d1dcc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-91fdc-predictor-5d77898ff9-wkvdq namespace: kserve-ci-e2e-test resourceVersion: "26269" uid: 6a5ec9e4-4350-4ed0-beca-d56c833516e7 kind: Event lastTimestamp: "2026-07-08T17:05:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T17:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:30Z" name: error-404-isvc-91fdc-predictor-5d77898ff9-wkvdq.18c05f62d0856452 namespace: kserve-ci-e2e-test resourceVersion: "26306" uid: 2ef67532-ef69-4fd6-8d55-3e81500d5b9f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-91fdc-predictor-5d77898ff9-wkvdq namespace: kserve-ci-e2e-test resourceVersion: "26269" uid: 6a5ec9e4-4350-4ed0-beca-d56c833516e7 kind: Event lastTimestamp: "2026-07-08T17:05:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T17:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:30Z" name: error-404-isvc-91fdc-predictor-5d77898ff9-wkvdq.18c05f62d0f65e27 namespace: kserve-ci-e2e-test resourceVersion: "26307" uid: 7cb0037e-2cc3-4dcd-9b6d-7aa151837127 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-91fdc-predictor-5d77898ff9-wkvdq namespace: kserve-ci-e2e-test resourceVersion: "26269" uid: 6a5ec9e4-4350-4ed0-beca-d56c833516e7 kind: Event lastTimestamp: "2026-07-08T17:05:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T17:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:30Z" name: error-404-isvc-91fdc-predictor-5d77898ff9-wkvdq.18c05f62d100dd9d namespace: kserve-ci-e2e-test resourceVersion: "26308" uid: 89abd958-9e8e-484f-9b5c-4c6ffd04cf85 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-91fdc-predictor-5d77898ff9-wkvdq namespace: kserve-ci-e2e-test resourceVersion: "26269" uid: 6a5ec9e4-4350-4ed0-beca-d56c833516e7 kind: Event lastTimestamp: "2026-07-08T17:05:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T17:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:30Z" name: error-404-isvc-91fdc-predictor-5d77898ff9-wkvdq.18c05f62d41900b5 namespace: kserve-ci-e2e-test resourceVersion: "26310" uid: 5fcdd8cb-28c4-44f9-afa4-a53f8e3b47ae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-91fdc-predictor-5d77898ff9-wkvdq namespace: kserve-ci-e2e-test resourceVersion: "26269" uid: 6a5ec9e4-4350-4ed0-beca-d56c833516e7 kind: Event lastTimestamp: "2026-07-08T17:05:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:30Z" name: error-404-isvc-91fdc-predictor-5d77898ff9-wkvdq.18c05f62d488f719 namespace: kserve-ci-e2e-test resourceVersion: "26311" uid: b0610c72-ee27-4272-b802-ba936a9c4b07 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-08T17:05:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-91fdc-predictor-5d77898ff9-wkvdq namespace: kserve-ci-e2e-test resourceVersion: "26269" uid: 6a5ec9e4-4350-4ed0-beca-d56c833516e7 kind: Event lastTimestamp: "2026-07-08T17:14:47Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:05:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:14:47Z" name: error-404-isvc-91fdc-predictor-5d77898ff9-wkvdq.18c05f630fd02faa namespace: kserve-ci-e2e-test resourceVersion: "30393" uid: 646423d7-5a22-4829-8f85-f1f06c4a0971 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:14:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-91fdc-predictor-5d77898ff9-wkvdq namespace: kserve-ci-e2e-test resourceVersion: "26269" uid: 6a5ec9e4-4350-4ed0-beca-d56c833516e7 kind: Event lastTimestamp: "2026-07-08T17:14:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T17:14:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:14:44Z" name: error-404-isvc-91fdc-predictor-5d77898ff9-wkvdq.18c05fe3dcfa2552 namespace: kserve-ci-e2e-test resourceVersion: "30371" uid: 927d4158-f34f-4b97-ad1d-9c1e5df659fe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:14:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-91fdc-predictor-5d77898ff9-wkvdq namespace: kserve-ci-e2e-test resourceVersion: "26269" uid: 6a5ec9e4-4350-4ed0-beca-d56c833516e7 kind: Event lastTimestamp: "2026-07-08T17:14:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:14:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:14:44Z" name: error-404-isvc-91fdc-predictor-5d77898ff9-wkvdq.18c05fe3dcfa9108 namespace: kserve-ci-e2e-test resourceVersion: "30370" uid: fdb299c5-d3ca-484b-bab3-185f7eaed3b9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:14:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-91fdc-predictor-5d77898ff9-wkvdq namespace: kserve-ci-e2e-test resourceVersion: "26269" uid: 6a5ec9e4-4350-4ed0-beca-d56c833516e7 kind: Event lastTimestamp: "2026-07-08T17:14:47Z" message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp 10.133.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:14:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:14:47Z" name: error-404-isvc-91fdc-predictor-5d77898ff9-wkvdq.18c05fe4842ce6bc namespace: kserve-ci-e2e-test resourceVersion: "30392" uid: 68b48501-7524-4329-b4b9-68f08b362c43 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-91fdc-predictor-5d77898ff9 namespace: kserve-ci-e2e-test resourceVersion: "26260" uid: 7117a958-646a-470c-9127-23d35b2d181c kind: Event lastTimestamp: "2026-07-08T17:05:29Z" message: 'Created pod: error-404-isvc-91fdc-predictor-5d77898ff9-wkvdq' metadata: creationTimestamp: "2026-07-08T17:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:05:29Z" name: error-404-isvc-91fdc-predictor-5d77898ff9.18c05f62b186101c namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 136cd43a-00df-40a2-a40a-f8c8a11cea05 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-91fdc-predictor namespace: kserve-ci-e2e-test resourceVersion: "26258" uid: 191387d9-c92e-446f-8d2d-fad2c67e5b19 kind: Event lastTimestamp: "2026-07-08T17:05:29Z" message: Scaled up replica set error-404-isvc-91fdc-predictor-5d77898ff9 from 0 to 1 metadata: creationTimestamp: "2026-07-08T17:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:05:29Z" name: error-404-isvc-91fdc-predictor.18c05f62afd6f8cd namespace: kserve-ci-e2e-test resourceVersion: "26261" uid: b5fd3757-e9a1-41ce-b7e9-6d1ed37ec119 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T17:05:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-91fdc-predictor namespace: kserve-ci-e2e-test resourceVersion: "26259" uid: 0f0c187c-e3a6-4f4e-89ed-a72351e384cd kind: Event lastTimestamp: "2026-07-08T17:06:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:06:14Z" name: error-404-isvc-91fdc-predictor.18c05f662e532690 namespace: kserve-ci-e2e-test resourceVersion: "26741" uid: b86df10b-7e6b-4eec-9c2e-56f7068e3346 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T17:05:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-91fdc-predictor namespace: kserve-ci-e2e-test resourceVersion: "26259" uid: 0f0c187c-e3a6-4f4e-89ed-a72351e384cd kind: Event lastTimestamp: "2026-07-08T17:06:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:06:14Z" name: error-404-isvc-91fdc-predictor.18c05f662ebcbde4 namespace: kserve-ci-e2e-test resourceVersion: "26742" uid: da31ff84-27b7-4daa-bc78-b1223b0f3ad1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:06:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-91fdc-predictor namespace: kserve-ci-e2e-test resourceVersion: "26443" uid: 0f0c187c-e3a6-4f4e-89ed-a72351e384cd kind: Event lastTimestamp: "2026-07-08T17:06:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:06:44Z" name: error-404-isvc-91fdc-predictor.18c05f70ab476685 namespace: kserve-ci-e2e-test resourceVersion: "27022" uid: bb50ff77-210f-4aaa-8b79-6f4949160136 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:06:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-91fdc-predictor namespace: kserve-ci-e2e-test resourceVersion: "26443" uid: 0f0c187c-e3a6-4f4e-89ed-a72351e384cd kind: Event lastTimestamp: "2026-07-08T17:06:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:06:44Z" name: error-404-isvc-91fdc-predictor.18c05f70abb90d47 namespace: kserve-ci-e2e-test resourceVersion: "27023" uid: a2308c77-3734-420c-83e2-41e93c2d2147 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T17:05:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-91fdc namespace: kserve-ci-e2e-test resourceVersion: "26245" uid: 7198b06e-4b2d-47a0-99a7-c1e11be22974 kind: Event lastTimestamp: "2026-07-08T17:06:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T17:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:06:17Z" name: error-404-isvc-91fdc.18c05f62afd6df12 namespace: kserve-ci-e2e-test resourceVersion: "26768" uid: 2d18ac44-b98b-4f25-945f-08d4c14ad6c5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-91fdc namespace: kserve-ci-e2e-test resourceVersion: "26245" uid: 7198b06e-4b2d-47a0-99a7-c1e11be22974 kind: Event lastTimestamp: "2026-07-08T17:05:29Z" message: 'Failed to update status for InferenceService "error-404-isvc-91fdc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-91fdc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T17:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:05:29Z" name: error-404-isvc-91fdc.18c05f62b02e53f4 namespace: kserve-ci-e2e-test resourceVersion: "26264" uid: 2dffeea3-865d-4f9d-9ab4-fb3ffa1803f7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-91fdc namespace: kserve-ci-e2e-test resourceVersion: "26245" uid: 7198b06e-4b2d-47a0-99a7-c1e11be22974 kind: Event lastTimestamp: "2026-07-08T17:05:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-91fdc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T17:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:05:29Z" name: error-404-isvc-91fdc.18c05f62b02ea871 namespace: kserve-ci-e2e-test resourceVersion: "26265" uid: c4cf3de2-8923-4525-815b-db038fcdf81e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:06:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-91fdc namespace: kserve-ci-e2e-test resourceVersion: "26766" uid: 7198b06e-4b2d-47a0-99a7-c1e11be22974 kind: Event lastTimestamp: "2026-07-08T17:06:17Z" message: InferenceService [error-404-isvc-91fdc] is Ready metadata: creationTimestamp: "2026-07-08T17:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:06:17Z" name: error-404-isvc-91fdc.18c05f6dcb31a193 namespace: kserve-ci-e2e-test resourceVersion: "26767" uid: a74be84f-9a54-46a1-a00f-911f54510239 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:42:49.480984Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9 namespace: kserve-ci-e2e-test resourceVersion: "14244" uid: 845cf57f-2f64-42d4-a113-a29138a8f27d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9 to ip-10-0-138-233.ec2.internal metadata: creationTimestamp: "2026-07-08T16:42:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:42:49Z" name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9.18c05e25f0887448 namespace: kserve-ci-e2e-test resourceVersion: "14249" uid: 33e20c9f-df5a-40d7-8929-4fa129bb2899 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:49Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9 namespace: kserve-ci-e2e-test resourceVersion: "14246" uid: 845cf57f-2f64-42d4-a113-a29138a8f27d kind: Event lastTimestamp: "2026-07-08T16:42:49Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-ed4fa-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T16:42:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:42:49Z" name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9.18c05e25fcb25884 namespace: kserve-ci-e2e-test resourceVersion: "14282" uid: 21c0ce42-45da-4ee5-9d96-dc4eb9f60d7d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:50Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9 namespace: kserve-ci-e2e-test resourceVersion: "14254" uid: 845cf57f-2f64-42d4-a113-a29138a8f27d kind: Event lastTimestamp: "2026-07-08T16:42:50Z" message: Add eth0 [10.133.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:42:50Z" name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9.18c05e262fd9932c namespace: kserve-ci-e2e-test resourceVersion: "14324" uid: 6f08caee-a978-4c04-9bae-fdfaa5443e0b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9 namespace: kserve-ci-e2e-test resourceVersion: "14246" uid: 845cf57f-2f64-42d4-a113-a29138a8f27d kind: Event lastTimestamp: "2026-07-08T16:42:50Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1707" metadata: creationTimestamp: "2026-07-08T16:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:42:50Z" name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9.18c05e26315e0732 namespace: kserve-ci-e2e-test resourceVersion: "14326" uid: 83257b59-8513-4e0e-8537-e4e0bffc88f5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9 namespace: kserve-ci-e2e-test resourceVersion: "14246" uid: 845cf57f-2f64-42d4-a113-a29138a8f27d kind: Event lastTimestamp: "2026-07-08T16:43:14Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1707" in 24.073s (24.073s including waiting). Image size: 1334629487 bytes.' metadata: creationTimestamp: "2026-07-08T16:43:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:14Z" name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9.18c05e2bcc44b1ff namespace: kserve-ci-e2e-test resourceVersion: "14556" uid: 01f64ce1-f546-4bca-bca8-8cbc41810b49 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9 namespace: kserve-ci-e2e-test resourceVersion: "14246" uid: 845cf57f-2f64-42d4-a113-a29138a8f27d kind: Event lastTimestamp: "2026-07-08T16:43:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:43:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:22Z" name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9.18c05e2db63f70fd namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 4a38bdcb-ef03-43cf-ae2e-709b07bb6e31 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9 namespace: kserve-ci-e2e-test resourceVersion: "14246" uid: 845cf57f-2f64-42d4-a113-a29138a8f27d kind: Event lastTimestamp: "2026-07-08T16:43:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:43:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:22Z" name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9.18c05e2db76e3d2c namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: 7d0811d8-99b9-422f-a448-51e69b286789 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9 namespace: kserve-ci-e2e-test resourceVersion: "14246" uid: 845cf57f-2f64-42d4-a113-a29138a8f27d kind: Event lastTimestamp: "2026-07-08T16:43:22Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-08T16:43:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:22Z" name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9.18c05e2db779f168 namespace: kserve-ci-e2e-test resourceVersion: "14631" uid: a8a253bb-bf37-4665-bb77-67e54ca1fba9 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9 namespace: kserve-ci-e2e-test resourceVersion: "14246" uid: 845cf57f-2f64-42d4-a113-a29138a8f27d kind: Event lastTimestamp: "2026-07-08T16:43:25Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.497s (2.497s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-08T16:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:25Z" name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9.18c05e2e4c56e139 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: c0fad235-4cac-4192-a08e-4a31e31ef920 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9 namespace: kserve-ci-e2e-test resourceVersion: "14246" uid: 845cf57f-2f64-42d4-a113-a29138a8f27d kind: Event lastTimestamp: "2026-07-08T16:43:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:25Z" name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9.18c05e2e50715b6b namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: 36600c14-0212-47a4-a03a-bef29b5e6ce5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9 namespace: kserve-ci-e2e-test resourceVersion: "14246" uid: 845cf57f-2f64-42d4-a113-a29138a8f27d kind: Event lastTimestamp: "2026-07-08T16:43:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:25Z" name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9.18c05e2e5179e887 namespace: kserve-ci-e2e-test resourceVersion: "14670" uid: 013a0f43-da6c-4a2e-a361-234d87cd75b3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-08T16:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9 namespace: kserve-ci-e2e-test resourceVersion: "14246" uid: 845cf57f-2f64-42d4-a113-a29138a8f27d kind: Event lastTimestamp: "2026-07-08T16:44:42Z" message: 'Readiness probe failed: dial tcp 10.133.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:42Z" name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9.18c05e2e7c9637b3 namespace: kserve-ci-e2e-test resourceVersion: "15501" uid: a33c90e3-9361-484b-981d-77ed6b060e59 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9 namespace: kserve-ci-e2e-test resourceVersion: "14246" uid: 845cf57f-2f64-42d4-a113-a29138a8f27d kind: Event lastTimestamp: "2026-07-08T16:44:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:39Z" name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9.18c05e3f8cd36a96 namespace: kserve-ci-e2e-test resourceVersion: "15431" uid: 1a408305-5cfe-45ec-b68b-820e8c66d6c7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9 namespace: kserve-ci-e2e-test resourceVersion: "14246" uid: 845cf57f-2f64-42d4-a113-a29138a8f27d kind: Event lastTimestamp: "2026-07-08T16:44:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:39Z" name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9.18c05e3f8cd40caf namespace: kserve-ci-e2e-test resourceVersion: "15434" uid: f7b2c515-377a-4caa-a6d8-c2fe16717c12 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9 namespace: kserve-ci-e2e-test resourceVersion: "14246" uid: 845cf57f-2f64-42d4-a113-a29138a8f27d kind: Event lastTimestamp: "2026-07-08T16:44:42Z" message: 'Readiness probe failed: Get "https://10.133.0.21:8643/healthz": dial tcp 10.133.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:44:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:42Z" name: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9.18c05e402ef0034c namespace: kserve-ci-e2e-test resourceVersion: "15500" uid: 81542439-08fa-400a-998c-0d888000188e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ed4fa-predictor-695ddbc98f namespace: kserve-ci-e2e-test resourceVersion: "14236" uid: 5eb2aa2a-f702-4fdc-a519-f7a410172691 kind: Event lastTimestamp: "2026-07-08T16:42:49Z" message: 'Created pod: error-404-isvc-ed4fa-predictor-695ddbc98f-tbbv9' metadata: creationTimestamp: "2026-07-08T16:42:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:42:49Z" name: error-404-isvc-ed4fa-predictor-695ddbc98f.18c05e25eff32e90 namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: d8ef26fc-11c2-4382-9e80-e234bb4fef60 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ed4fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "14233" uid: 0d2ad816-0d81-4c0f-904a-89cac07a2628 kind: Event lastTimestamp: "2026-07-08T16:42:49Z" message: Scaled up replica set error-404-isvc-ed4fa-predictor-695ddbc98f from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:42:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:42:49Z" name: error-404-isvc-ed4fa-predictor.18c05e25ee3d323c namespace: kserve-ci-e2e-test resourceVersion: "14237" uid: e244112e-24d2-4fca-b410-6020685a6b97 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:43:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ed4fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "14235" uid: 1dec7c0b-96fe-477d-aff0-58f570975b79 kind: Event lastTimestamp: "2026-07-08T16:43:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:43:49Z" name: error-404-isvc-ed4fa-predictor.18c05e296cd319cd namespace: kserve-ci-e2e-test resourceVersion: "14878" uid: e4608a3d-ea57-42c3-bf47-9fdaa12b7ada reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:43:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ed4fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "14235" uid: 1dec7c0b-96fe-477d-aff0-58f570975b79 kind: Event lastTimestamp: "2026-07-08T16:43:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:43:49Z" name: error-404-isvc-ed4fa-predictor.18c05e296d31c4e1 namespace: kserve-ci-e2e-test resourceVersion: "14879" uid: 2014caf2-8ab6-4eff-b27c-83ce882df43b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:44:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ed4fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "14471" uid: 1dec7c0b-96fe-477d-aff0-58f570975b79 kind: Event lastTimestamp: "2026-07-08T16:44:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:44:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:44:34Z" name: error-404-isvc-ed4fa-predictor.18c05e3768a5198b namespace: kserve-ci-e2e-test resourceVersion: "15301" uid: c0989816-bd9a-4738-8697-c5d694b92512 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:44:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ed4fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "14471" uid: 1dec7c0b-96fe-477d-aff0-58f570975b79 kind: Event lastTimestamp: "2026-07-08T16:44:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:44:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:44:34Z" name: error-404-isvc-ed4fa-predictor.18c05e3769164bfd namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: cfbe9354-0222-42fb-885a-97c86dad3d44 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:42:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ed4fa namespace: kserve-ci-e2e-test resourceVersion: "14218" uid: 87d68d97-f1f0-4465-8c68-9dd018845207 kind: Event lastTimestamp: "2026-07-08T16:44:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:42:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:44:22Z" name: error-404-isvc-ed4fa.18c05e25eeb15538 namespace: kserve-ci-e2e-test resourceVersion: "15161" uid: 7ae0dad5-8d98-4577-b38c-bcadf88d0de7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ed4fa namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 87d68d97-f1f0-4465-8c68-9dd018845207 kind: Event lastTimestamp: "2026-07-08T16:44:22Z" message: InferenceService [error-404-isvc-ed4fa] is Ready metadata: creationTimestamp: "2026-07-08T16:44:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:44:22Z" name: error-404-isvc-ed4fa.18c05e3b8c428b9f namespace: kserve-ci-e2e-test resourceVersion: "15160" uid: dd9acef1-67e8-4cf7-a212-8a774ddfb5d9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:55:44.589969Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f6dd6-predictor-9b494c85-f5c49 namespace: kserve-ci-e2e-test resourceVersion: "21253" uid: 28c51aa0-a630-45c2-9cdd-4fa2361d373c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f6dd6-predictor-9b494c85-f5c49 to ip-10-0-138-233.ec2.internal metadata: creationTimestamp: "2026-07-08T16:55:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:55:44Z" name: error-404-isvc-f6dd6-predictor-9b494c85-f5c49.18c05eda68a0ea74 namespace: kserve-ci-e2e-test resourceVersion: "21257" uid: 43ca4c95-d159-421c-8d75-b6554c0b93c8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:44Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f6dd6-predictor-9b494c85-f5c49 namespace: kserve-ci-e2e-test resourceVersion: "21255" uid: 28c51aa0-a630-45c2-9cdd-4fa2361d373c kind: Event lastTimestamp: "2026-07-08T16:55:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-f6dd6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T16:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:44Z" name: error-404-isvc-f6dd6-predictor-9b494c85-f5c49.18c05eda71286162 namespace: kserve-ci-e2e-test resourceVersion: "21268" uid: fadcfcb2-9590-4301-8246-38c4d9417cdc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:45Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f6dd6-predictor-9b494c85-f5c49 namespace: kserve-ci-e2e-test resourceVersion: "21263" uid: 28c51aa0-a630-45c2-9cdd-4fa2361d373c kind: Event lastTimestamp: "2026-07-08T16:55:45Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:55:45Z" name: error-404-isvc-f6dd6-predictor-9b494c85-f5c49.18c05edaa5f8a624 namespace: kserve-ci-e2e-test resourceVersion: "21289" uid: f3cafa6b-7b31-487b-90d0-bdd9e283c529 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f6dd6-predictor-9b494c85-f5c49 namespace: kserve-ci-e2e-test resourceVersion: "21255" uid: 28c51aa0-a630-45c2-9cdd-4fa2361d373c kind: Event lastTimestamp: "2026-07-08T16:55:45Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1707" already present on machine metadata: creationTimestamp: "2026-07-08T16:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:45Z" name: error-404-isvc-f6dd6-predictor-9b494c85-f5c49.18c05edaa75c8ae8 namespace: kserve-ci-e2e-test resourceVersion: "21291" uid: 83564c2f-419a-4c59-b7b9-e4ac2aa69bb5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f6dd6-predictor-9b494c85-f5c49 namespace: kserve-ci-e2e-test resourceVersion: "21255" uid: 28c51aa0-a630-45c2-9cdd-4fa2361d373c kind: Event lastTimestamp: "2026-07-08T16:55:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:45Z" name: error-404-isvc-f6dd6-predictor-9b494c85-f5c49.18c05edaaa4246d1 namespace: kserve-ci-e2e-test resourceVersion: "21292" uid: add24a0e-cb3c-46b5-b4f6-7089e5234837 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f6dd6-predictor-9b494c85-f5c49 namespace: kserve-ci-e2e-test resourceVersion: "21255" uid: 28c51aa0-a630-45c2-9cdd-4fa2361d373c kind: Event lastTimestamp: "2026-07-08T16:55:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:45Z" name: error-404-isvc-f6dd6-predictor-9b494c85-f5c49.18c05edaaade09f5 namespace: kserve-ci-e2e-test resourceVersion: "21294" uid: e03c88f9-ebf5-4004-b171-49695fe5248d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f6dd6-predictor-9b494c85-f5c49 namespace: kserve-ci-e2e-test resourceVersion: "21255" uid: 28c51aa0-a630-45c2-9cdd-4fa2361d373c kind: Event lastTimestamp: "2026-07-08T16:55:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:45Z" name: error-404-isvc-f6dd6-predictor-9b494c85-f5c49.18c05edaaae9315b namespace: kserve-ci-e2e-test resourceVersion: "21295" uid: b0847412-3621-4361-a972-c04408b430ef reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f6dd6-predictor-9b494c85-f5c49 namespace: kserve-ci-e2e-test resourceVersion: "21255" uid: 28c51aa0-a630-45c2-9cdd-4fa2361d373c kind: Event lastTimestamp: "2026-07-08T16:55:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:45Z" name: error-404-isvc-f6dd6-predictor-9b494c85-f5c49.18c05edaadda143a namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 10f3a776-1f68-46b5-9b75-3965b48a841b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f6dd6-predictor-9b494c85-f5c49 namespace: kserve-ci-e2e-test resourceVersion: "21255" uid: 28c51aa0-a630-45c2-9cdd-4fa2361d373c kind: Event lastTimestamp: "2026-07-08T16:55:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:45Z" name: error-404-isvc-f6dd6-predictor-9b494c85-f5c49.18c05edaae53ef27 namespace: kserve-ci-e2e-test resourceVersion: "21300" uid: 76fc9fba-4041-4a3e-af72-db6bd0b96e76 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:55:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f6dd6-predictor-9b494c85-f5c49 namespace: kserve-ci-e2e-test resourceVersion: "21255" uid: 28c51aa0-a630-45c2-9cdd-4fa2361d373c kind: Event lastTimestamp: "2026-07-08T16:56:23Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:55:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:56:23Z" name: error-404-isvc-f6dd6-predictor-9b494c85-f5c49.18c05edb031e4c79 namespace: kserve-ci-e2e-test resourceVersion: "21679" uid: 83407047-cc23-4662-9bc7-2fa30ed669aa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f6dd6-predictor-9b494c85-f5c49 namespace: kserve-ci-e2e-test resourceVersion: "21255" uid: 28c51aa0-a630-45c2-9cdd-4fa2361d373c kind: Event lastTimestamp: "2026-07-08T17:04:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:59Z" name: error-404-isvc-f6dd6-predictor-9b494c85-f5c49.18c05f5b9884f942 namespace: kserve-ci-e2e-test resourceVersion: "25792" uid: e65c268c-aac7-452b-bb71-6b4094c08e49 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f6dd6-predictor-9b494c85-f5c49 namespace: kserve-ci-e2e-test resourceVersion: "21255" uid: 28c51aa0-a630-45c2-9cdd-4fa2361d373c kind: Event lastTimestamp: "2026-07-08T17:04:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:59Z" name: error-404-isvc-f6dd6-predictor-9b494c85-f5c49.18c05f5b98857951 namespace: kserve-ci-e2e-test resourceVersion: "25793" uid: e98b9269-625a-46e6-a9ab-feab9213b541 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f6dd6-predictor-9b494c85 namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: 9a5e1ef5-1e44-45c9-a0bb-989aae6266ac kind: Event lastTimestamp: "2026-07-08T16:55:44Z" message: 'Created pod: error-404-isvc-f6dd6-predictor-9b494c85-f5c49' metadata: creationTimestamp: "2026-07-08T16:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:44Z" name: error-404-isvc-f6dd6-predictor-9b494c85.18c05eda681eb6d8 namespace: kserve-ci-e2e-test resourceVersion: "21254" uid: c8565a85-d7e0-4bec-8be1-639394811ff8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f6dd6-predictor namespace: kserve-ci-e2e-test resourceVersion: "21242" uid: e3c52699-fad2-4eaf-a747-c360fa35548c kind: Event lastTimestamp: "2026-07-08T16:55:44Z" message: Scaled up replica set error-404-isvc-f6dd6-predictor-9b494c85 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:44Z" name: error-404-isvc-f6dd6-predictor.18c05eda6647d00e namespace: kserve-ci-e2e-test resourceVersion: "21247" uid: 6521499d-d7fd-4eac-a590-3a3cfc89c8a5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:55:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f6dd6-predictor namespace: kserve-ci-e2e-test resourceVersion: "21243" uid: 0a0fdf7e-d7ce-4fb0-a614-5f96f6d006f8 kind: Event lastTimestamp: "2026-07-08T16:56:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:56:29Z" name: error-404-isvc-f6dd6-predictor.18c05edde49b1433 namespace: kserve-ci-e2e-test resourceVersion: "21731" uid: 217b8919-7c46-412a-a81b-66a1033eceea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:55:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f6dd6-predictor namespace: kserve-ci-e2e-test resourceVersion: "21243" uid: 0a0fdf7e-d7ce-4fb0-a614-5f96f6d006f8 kind: Event lastTimestamp: "2026-07-08T16:56:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:56:29Z" name: error-404-isvc-f6dd6-predictor.18c05edde5114222 namespace: kserve-ci-e2e-test resourceVersion: "21732" uid: 0dd9c2a2-e84e-434b-90a7-9e8a268160fc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:56:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f6dd6-predictor namespace: kserve-ci-e2e-test resourceVersion: "21449" uid: 0a0fdf7e-d7ce-4fb0-a614-5f96f6d006f8 kind: Event lastTimestamp: "2026-07-08T16:56:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:56:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:56:59Z" name: error-404-isvc-f6dd6-predictor.18c05ee861695801 namespace: kserve-ci-e2e-test resourceVersion: "22010" uid: 7b9aeffd-81a3-464f-a521-2640d27241e8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:56:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f6dd6-predictor namespace: kserve-ci-e2e-test resourceVersion: "21449" uid: 0a0fdf7e-d7ce-4fb0-a614-5f96f6d006f8 kind: Event lastTimestamp: "2026-07-08T16:56:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:56:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:56:59Z" name: error-404-isvc-f6dd6-predictor.18c05ee861d88a70 namespace: kserve-ci-e2e-test resourceVersion: "22011" uid: 9a886cd9-12ab-4be8-9b63-f05a83e5fc7c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:55:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f6dd6 namespace: kserve-ci-e2e-test resourceVersion: "21225" uid: 0fb8e73c-29e3-4dd6-80db-e68ffd86b83d kind: Event lastTimestamp: "2026-07-08T16:56:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:56:33Z" name: error-404-isvc-f6dd6.18c05eda6643d895 namespace: kserve-ci-e2e-test resourceVersion: "21772" uid: f1e62eeb-2b0b-4aca-845f-8f30ed9c2a06 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f6dd6 namespace: kserve-ci-e2e-test resourceVersion: "21225" uid: 0fb8e73c-29e3-4dd6-80db-e68ffd86b83d kind: Event lastTimestamp: "2026-07-08T16:55:44Z" message: 'Failed to update status for InferenceService "error-404-isvc-f6dd6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f6dd6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:55:44Z" name: error-404-isvc-f6dd6.18c05eda66f93540 namespace: kserve-ci-e2e-test resourceVersion: "21250" uid: 99e8770f-bce2-42cf-8d41-f78565d34731 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f6dd6 namespace: kserve-ci-e2e-test resourceVersion: "21225" uid: 0fb8e73c-29e3-4dd6-80db-e68ffd86b83d kind: Event lastTimestamp: "2026-07-08T16:55:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f6dd6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:55:44Z" name: error-404-isvc-f6dd6.18c05eda66f9b413 namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: a970619a-eae7-43c9-bcd7-6885e3b985be reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f6dd6 namespace: kserve-ci-e2e-test resourceVersion: "21769" uid: 0fb8e73c-29e3-4dd6-80db-e68ffd86b83d kind: Event lastTimestamp: "2026-07-08T16:56:33Z" message: InferenceService [error-404-isvc-f6dd6] is Ready metadata: creationTimestamp: "2026-07-08T16:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:56:33Z" name: error-404-isvc-f6dd6.18c05ee5bf13416e namespace: kserve-ci-e2e-test resourceVersion: "21770" uid: 37cab9e9-980e-4e4e-8ee0-c23751a215c9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:45:19.507004Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fabf2-predictor-54645d4667-9pmq6 namespace: kserve-ci-e2e-test resourceVersion: "15981" uid: 8240b020-722e-4446-9dc9-666e3e2b2bab kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-fabf2-predictor-54645d4667-9pmq6 to ip-10-0-138-233.ec2.internal metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:45:19Z" name: error-404-isvc-fabf2-predictor-54645d4667-9pmq6.18c05e48dec7d658 namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: 0119bc1d-e00d-49a8-a8bd-37698ccfcecd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:19Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fabf2-predictor-54645d4667-9pmq6 namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: 8240b020-722e-4446-9dc9-666e3e2b2bab kind: Event lastTimestamp: "2026-07-08T16:45:19Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:45:19Z" name: error-404-isvc-fabf2-predictor-54645d4667-9pmq6.18c05e48f848b1f4 namespace: kserve-ci-e2e-test resourceVersion: "16010" uid: 92490649-a859-441a-bb0d-cb31006f7115 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fabf2-predictor-54645d4667-9pmq6 namespace: kserve-ci-e2e-test resourceVersion: "15985" uid: 8240b020-722e-4446-9dc9-666e3e2b2bab kind: Event lastTimestamp: "2026-07-08T16:45:19Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1707" already present on machine metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:19Z" name: error-404-isvc-fabf2-predictor-54645d4667-9pmq6.18c05e48f9c9db1a namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: f882dac7-653b-487d-a965-b229594c513b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fabf2-predictor-54645d4667-9pmq6 namespace: kserve-ci-e2e-test resourceVersion: "15985" uid: 8240b020-722e-4446-9dc9-666e3e2b2bab kind: Event lastTimestamp: "2026-07-08T16:45:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:45:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:20Z" name: error-404-isvc-fabf2-predictor-54645d4667-9pmq6.18c05e48fc7e4c70 namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: b7d3b3a0-cf3b-488b-a84e-d2ca68a5e0ac reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fabf2-predictor-54645d4667-9pmq6 namespace: kserve-ci-e2e-test resourceVersion: "15985" uid: 8240b020-722e-4446-9dc9-666e3e2b2bab kind: Event lastTimestamp: "2026-07-08T16:45:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:45:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:20Z" name: error-404-isvc-fabf2-predictor-54645d4667-9pmq6.18c05e48fce2c894 namespace: kserve-ci-e2e-test resourceVersion: "16016" uid: a0e05950-9c1f-4074-a4dd-6c9b433b1fda reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fabf2-predictor-54645d4667-9pmq6 namespace: kserve-ci-e2e-test resourceVersion: "15985" uid: 8240b020-722e-4446-9dc9-666e3e2b2bab kind: Event lastTimestamp: "2026-07-08T16:45:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:45:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:20Z" name: error-404-isvc-fabf2-predictor-54645d4667-9pmq6.18c05e48fcecc37c namespace: kserve-ci-e2e-test resourceVersion: "16017" uid: d4b1ff04-1795-4652-9e62-868d2ecf7563 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fabf2-predictor-54645d4667-9pmq6 namespace: kserve-ci-e2e-test resourceVersion: "15985" uid: 8240b020-722e-4446-9dc9-666e3e2b2bab kind: Event lastTimestamp: "2026-07-08T16:45:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:45:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:20Z" name: error-404-isvc-fabf2-predictor-54645d4667-9pmq6.18c05e48ffe53be9 namespace: kserve-ci-e2e-test resourceVersion: "16022" uid: e7efb210-21ff-4353-8dec-1031719c4610 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fabf2-predictor-54645d4667-9pmq6 namespace: kserve-ci-e2e-test resourceVersion: "15985" uid: 8240b020-722e-4446-9dc9-666e3e2b2bab kind: Event lastTimestamp: "2026-07-08T16:45:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:45:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:20Z" name: error-404-isvc-fabf2-predictor-54645d4667-9pmq6.18c05e490055714d namespace: kserve-ci-e2e-test resourceVersion: "16024" uid: 48cf8cdd-8f45-46b7-8f12-6967b1f1c9ae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:45:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fabf2-predictor-54645d4667-9pmq6 namespace: kserve-ci-e2e-test resourceVersion: "15985" uid: 8240b020-722e-4446-9dc9-666e3e2b2bab kind: Event lastTimestamp: "2026-07-08T16:45:57Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:57Z" name: error-404-isvc-fabf2-predictor-54645d4667-9pmq6.18c05e49615bb8e1 namespace: kserve-ci-e2e-test resourceVersion: "16406" uid: f75a8fae-fd56-47ec-b5da-d8662648ecfd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fabf2-predictor-54645d4667-9pmq6 namespace: kserve-ci-e2e-test resourceVersion: "15985" uid: 8240b020-722e-4446-9dc9-666e3e2b2bab kind: Event lastTimestamp: "2026-07-08T16:54:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:34Z" name: error-404-isvc-fabf2-predictor-54645d4667-9pmq6.18c05eca08e3e1c1 namespace: kserve-ci-e2e-test resourceVersion: "20274" uid: db1c10a5-c58e-492a-89c1-7fb087543a51 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fabf2-predictor-54645d4667-9pmq6 namespace: kserve-ci-e2e-test resourceVersion: "15985" uid: 8240b020-722e-4446-9dc9-666e3e2b2bab kind: Event lastTimestamp: "2026-07-08T16:54:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:34Z" name: error-404-isvc-fabf2-predictor-54645d4667-9pmq6.18c05eca08e4ddda namespace: kserve-ci-e2e-test resourceVersion: "20276" uid: ffb7a1e6-6f10-4750-ad16-e1f890d588a4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-fabf2-predictor-54645d4667 namespace: kserve-ci-e2e-test resourceVersion: "15971" uid: fe1bf58c-4917-4e75-a7fc-556c8d0fbd86 kind: Event lastTimestamp: "2026-07-08T16:45:19Z" message: 'Created pod: error-404-isvc-fabf2-predictor-54645d4667-9pmq6' metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:45:19Z" name: error-404-isvc-fabf2-predictor-54645d4667.18c05e48de11caef namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: af7ff376-8ec8-4960-a602-a55236d6c589 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-fabf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15968" uid: 4cc53d2d-39a9-4fb7-8802-849198f4ec65 kind: Event lastTimestamp: "2026-07-08T16:45:19Z" message: Scaled up replica set error-404-isvc-fabf2-predictor-54645d4667 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:45:19Z" name: error-404-isvc-fabf2-predictor.18c05e48dc31ce16 namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: 59d6b659-ada0-4aee-83ac-6072804b4e55 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:45:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fabf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: ad067358-676a-4c80-ad26-fa98e87b061e kind: Event lastTimestamp: "2026-07-08T16:46:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:04Z" name: error-404-isvc-fabf2-predictor.18c05e4c5ac50dcf namespace: kserve-ci-e2e-test resourceVersion: "16454" uid: 29ec0b90-8ec7-4dd1-af78-599ad59438b6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:45:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fabf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: ad067358-676a-4c80-ad26-fa98e87b061e kind: Event lastTimestamp: "2026-07-08T16:46:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:04Z" name: error-404-isvc-fabf2-predictor.18c05e4c5b3afd3d namespace: kserve-ci-e2e-test resourceVersion: "16455" uid: d2815c7e-6f5d-4e9d-8a46-4820f763ab18 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:45:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fabf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16179" uid: ad067358-676a-4c80-ad26-fa98e87b061e kind: Event lastTimestamp: "2026-07-08T16:46:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:49Z" name: error-404-isvc-fabf2-predictor.18c05e4fda09ff4b namespace: kserve-ci-e2e-test resourceVersion: "16843" uid: 23cccfcf-eaed-445f-aa36-79ca4b5fa888 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:45:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fabf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16179" uid: ad067358-676a-4c80-ad26-fa98e87b061e kind: Event lastTimestamp: "2026-07-08T16:46:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:49Z" name: error-404-isvc-fabf2-predictor.18c05e4fda7f9dd3 namespace: kserve-ci-e2e-test resourceVersion: "16844" uid: 3c62d7c8-af9b-47bd-b2e5-19acb1d5ee80 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:45:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fabf2 namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 1ddcbfc3-b4bc-4d45-a1a8-432223d33af4 kind: Event lastTimestamp: "2026-07-08T16:46:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:46:07Z" name: error-404-isvc-fabf2.18c05e48dc43f198 namespace: kserve-ci-e2e-test resourceVersion: "16499" uid: 4bf0ebb3-23f5-4a4a-bad8-ea6c06333610 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fabf2 namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 1ddcbfc3-b4bc-4d45-a1a8-432223d33af4 kind: Event lastTimestamp: "2026-07-08T16:45:19Z" message: 'Failed to update status for InferenceService "error-404-isvc-fabf2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fabf2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:45:19Z" name: error-404-isvc-fabf2.18c05e48dcfc149c namespace: kserve-ci-e2e-test resourceVersion: "15979" uid: 397c82e3-32a8-49a5-bbac-640b020bee54 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fabf2 namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 1ddcbfc3-b4bc-4d45-a1a8-432223d33af4 kind: Event lastTimestamp: "2026-07-08T16:45:19Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fabf2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:45:19Z" name: error-404-isvc-fabf2.18c05e48dcfc6c24 namespace: kserve-ci-e2e-test resourceVersion: "15982" uid: 7167c767-d1ed-43ff-a276-33d0c5bf3552 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fabf2 namespace: kserve-ci-e2e-test resourceVersion: "16496" uid: 1ddcbfc3-b4bc-4d45-a1a8-432223d33af4 kind: Event lastTimestamp: "2026-07-08T16:46:07Z" message: InferenceService [error-404-isvc-fabf2] is Ready metadata: creationTimestamp: "2026-07-08T16:46:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:46:07Z" name: error-404-isvc-fabf2.18c05e541ddd3379 namespace: kserve-ci-e2e-test resourceVersion: "16497" uid: e81bd223-1920-41ae-8fa0-607e0f7f3ffb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:42:49.116723Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt namespace: kserve-ci-e2e-test resourceVersion: "14162" uid: 6d0019fc-a462-41c4-a14f-05ce0b81ef57 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-85d568d89d-j22lt to ip-10-0-138-233.ec2.internal metadata: creationTimestamp: "2026-07-08T16:42:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:42:49Z" name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt.18c05e25dad25810 namespace: kserve-ci-e2e-test resourceVersion: "14167" uid: 5c060ccb-c6a4-4151-bf59-2f83a2f25b93 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:49Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt namespace: kserve-ci-e2e-test resourceVersion: "14164" uid: 6d0019fc-a462-41c4-a14f-05ce0b81ef57 kind: Event lastTimestamp: "2026-07-08T16:42:49Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T16:42:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:42:49Z" name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt.18c05e25ea903d56 namespace: kserve-ci-e2e-test resourceVersion: "14220" uid: 1ebda504-e749-44b4-bca2-0e0adda56404 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:50Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt namespace: kserve-ci-e2e-test resourceVersion: "14174" uid: 6d0019fc-a462-41c4-a14f-05ce0b81ef57 kind: Event lastTimestamp: "2026-07-08T16:42:50Z" message: Add eth0 [10.133.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:42:50Z" name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt.18c05e2618b99dd1 namespace: kserve-ci-e2e-test resourceVersion: "14311" uid: 097d8078-07fb-4cf0-b1d5-f67762c5c69d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt namespace: kserve-ci-e2e-test resourceVersion: "14164" uid: 6d0019fc-a462-41c4-a14f-05ce0b81ef57 kind: Event lastTimestamp: "2026-07-08T16:42:50Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:43b18792ad3c8c70fe63e50dee47838dba7fd6b00a2dcfe15c76c1dafc2c7ad8" already present on machine metadata: creationTimestamp: "2026-07-08T16:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:42:50Z" name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt.18c05e261a2d4f6c namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: f38c737e-d3ec-43ed-9609-20e104a08ec4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt namespace: kserve-ci-e2e-test resourceVersion: "14164" uid: 6d0019fc-a462-41c4-a14f-05ce0b81ef57 kind: Event lastTimestamp: "2026-07-08T16:42:50Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T16:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:42:50Z" name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt.18c05e261cc3af21 namespace: kserve-ci-e2e-test resourceVersion: "14315" uid: 3718f361-3e57-4b47-bbd0-b145eab67044 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt namespace: kserve-ci-e2e-test resourceVersion: "14164" uid: 6d0019fc-a462-41c4-a14f-05ce0b81ef57 kind: Event lastTimestamp: "2026-07-08T16:42:50Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T16:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:42:50Z" name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt.18c05e261d2acc96 namespace: kserve-ci-e2e-test resourceVersion: "14316" uid: d9c47c60-04c0-474e-92d3-19638b639fa9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt namespace: kserve-ci-e2e-test resourceVersion: "14164" uid: 6d0019fc-a462-41c4-a14f-05ce0b81ef57 kind: Event lastTimestamp: "2026-07-08T16:42:56Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1707" metadata: creationTimestamp: "2026-07-08T16:42:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:42:56Z" name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt.18c05e27751c7e1e namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: 599a1824-d5c5-4756-93a8-da7bf1e80bd6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt namespace: kserve-ci-e2e-test resourceVersion: "14164" uid: 6d0019fc-a462-41c4-a14f-05ce0b81ef57 kind: Event lastTimestamp: "2026-07-08T16:43:14Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1707" in 18.564s (18.564s including waiting). Image size: 1555648957 bytes.' metadata: creationTimestamp: "2026-07-08T16:43:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:14Z" name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt.18c05e2bc7a43e74 namespace: kserve-ci-e2e-test resourceVersion: "14554" uid: 78af528e-d170-421a-8cfb-a514d89d6685 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt namespace: kserve-ci-e2e-test resourceVersion: "14164" uid: 6d0019fc-a462-41c4-a14f-05ce0b81ef57 kind: Event lastTimestamp: "2026-07-08T16:43:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:43:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:22Z" name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt.18c05e2db6899f5e namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: 65859cf4-c22c-4c81-adaa-ec79683313d0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt namespace: kserve-ci-e2e-test resourceVersion: "14164" uid: 6d0019fc-a462-41c4-a14f-05ce0b81ef57 kind: Event lastTimestamp: "2026-07-08T16:43:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:43:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:22Z" name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt.18c05e2db7d798e0 namespace: kserve-ci-e2e-test resourceVersion: "14632" uid: 1a3cc94d-b812-443c-aad9-a3c5452561d8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt namespace: kserve-ci-e2e-test resourceVersion: "14164" uid: 6d0019fc-a462-41c4-a14f-05ce0b81ef57 kind: Event lastTimestamp: "2026-07-08T16:43:22Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-08T16:43:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:22Z" name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt.18c05e2db7e4982d namespace: kserve-ci-e2e-test resourceVersion: "14635" uid: a757a823-7efb-4ac4-95e4-f81a375cd8b0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt namespace: kserve-ci-e2e-test resourceVersion: "14164" uid: 6d0019fc-a462-41c4-a14f-05ce0b81ef57 kind: Event lastTimestamp: "2026-07-08T16:43:25Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.475s (2.475s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-08T16:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:25Z" name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt.18c05e2e4b6ea995 namespace: kserve-ci-e2e-test resourceVersion: "14663" uid: f675da0c-502b-4990-a902-bc8a9ca91593 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt namespace: kserve-ci-e2e-test resourceVersion: "14164" uid: 6d0019fc-a462-41c4-a14f-05ce0b81ef57 kind: Event lastTimestamp: "2026-07-08T16:43:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:25Z" name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt.18c05e2e506b32f3 namespace: kserve-ci-e2e-test resourceVersion: "14668" uid: f9ee2529-600d-419d-a853-81417208bca0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt namespace: kserve-ci-e2e-test resourceVersion: "14164" uid: 6d0019fc-a462-41c4-a14f-05ce0b81ef57 kind: Event lastTimestamp: "2026-07-08T16:43:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:25Z" name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt.18c05e2e517efe54 namespace: kserve-ci-e2e-test resourceVersion: "14672" uid: 199abcbc-67f6-4caa-aea9-a686b35f4cef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T16:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt namespace: kserve-ci-e2e-test resourceVersion: "14164" uid: 6d0019fc-a462-41c4-a14f-05ce0b81ef57 kind: Event lastTimestamp: "2026-07-08T16:45:22Z" message: 'Readiness probe failed: dial tcp 10.133.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:22Z" name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt.18c05e2e7cb84784 namespace: kserve-ci-e2e-test resourceVersion: "16050" uid: 0f024119-ebef-4f60-8c68-43cb13eef8b7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt namespace: kserve-ci-e2e-test resourceVersion: "14164" uid: 6d0019fc-a462-41c4-a14f-05ce0b81ef57 kind: Event lastTimestamp: "2026-07-08T16:45:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:19Z" name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt.18c05e48d6a34150 namespace: kserve-ci-e2e-test resourceVersion: "15939" uid: 4cb3fcae-2e79-4af8-b5b0-ecd5c34d354d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt namespace: kserve-ci-e2e-test resourceVersion: "14164" uid: 6d0019fc-a462-41c4-a14f-05ce0b81ef57 kind: Event lastTimestamp: "2026-07-08T16:45:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:19Z" name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt.18c05e48d6a3d51d namespace: kserve-ci-e2e-test resourceVersion: "15944" uid: 5a18498c-5980-405d-971a-4819dc89cb8f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt namespace: kserve-ci-e2e-test resourceVersion: "14164" uid: 6d0019fc-a462-41c4-a14f-05ce0b81ef57 kind: Event lastTimestamp: "2026-07-08T16:45:22Z" message: 'Readiness probe failed: Get "https://10.133.0.19:8643/healthz": dial tcp 10.133.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:45:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:22Z" name: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt.18c05e497f21d884 namespace: kserve-ci-e2e-test resourceVersion: "16047" uid: a2850607-de0b-4ff2-8ed3-63e3cb4231f7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-85d568d89d namespace: kserve-ci-e2e-test resourceVersion: "14155" uid: 30982970-bfab-47e0-8518-3418385d27f0 kind: Event lastTimestamp: "2026-07-08T16:42:49Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-85d568d89d-j22lt' metadata: creationTimestamp: "2026-07-08T16:42:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:42:49Z" name: isvc-sklearn-graph-1-predictor-85d568d89d.18c05e25da722450 namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 191b815f-fc86-4784-8d63-a257668602ec reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14154" uid: ea560a40-2756-411e-8e10-2ac5411f3c15 kind: Event lastTimestamp: "2026-07-08T16:42:49Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-85d568d89d from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:42:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:42:49Z" name: isvc-sklearn-graph-1-predictor.18c05e25d56eafa1 namespace: kserve-ci-e2e-test resourceVersion: "14156" uid: cc74dece-480f-40bb-aabd-ca36200cb638 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:43:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14166" uid: 94da96c1-fcd0-4b4a-8bf8-4ce2c2a137d3 kind: Event lastTimestamp: "2026-07-08T16:44:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:44:19Z" name: isvc-sklearn-graph-1-predictor.18c05e295b395a5f namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: 2bac7d32-c20a-41ee-8c4c-4e1029f68f48 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:43:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14166" uid: 94da96c1-fcd0-4b4a-8bf8-4ce2c2a137d3 kind: Event lastTimestamp: "2026-07-08T16:44:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:44:19Z" name: isvc-sklearn-graph-1-predictor.18c05e295ba8689a namespace: kserve-ci-e2e-test resourceVersion: "15122" uid: 696276ba-d679-4d91-ad8b-7f30d09b0204 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:44:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14462" uid: 94da96c1-fcd0-4b4a-8bf8-4ce2c2a137d3 kind: Event lastTimestamp: "2026-07-08T16:45:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:45:04Z" name: isvc-sklearn-graph-1-predictor.18c05e3e57be28f6 namespace: kserve-ci-e2e-test resourceVersion: "15706" uid: 31b2d4fc-020f-47f2-a1a5-207afdcee8fa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:44:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14462" uid: 94da96c1-fcd0-4b4a-8bf8-4ce2c2a137d3 kind: Event lastTimestamp: "2026-07-08T16:45:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:45:04Z" name: isvc-sklearn-graph-1-predictor.18c05e3e58343a14 namespace: kserve-ci-e2e-test resourceVersion: "15707" uid: 2e6a9f13-eed4-41b8-b716-2a3b6815ab46 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-08T16:42:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14142" uid: 31ce518f-7a87-4294-a7dd-24f9a7dd3470 kind: Event lastTimestamp: "2026-07-08T16:44:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:42:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:44:42Z" name: isvc-sklearn-graph-1.18c05e25db92faa1 namespace: kserve-ci-e2e-test resourceVersion: "15519" uid: 90205d73-7185-495b-aaf4-87bfedc1f0ad reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15516" uid: 31ce518f-7a87-4294-a7dd-24f9a7dd3470 kind: Event lastTimestamp: "2026-07-08T16:44:42Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-08T16:44:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:44:42Z" name: isvc-sklearn-graph-1.18c05e4039c93868 namespace: kserve-ci-e2e-test resourceVersion: "15517" uid: 9a899311-0272-4e49-b6f8-094bfd687d5d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:42:49.620998Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r namespace: kserve-ci-e2e-test resourceVersion: "14269" uid: 7a165d9e-b6f5-447d-be23-b27715bea795 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r to ip-10-0-138-233.ec2.internal metadata: creationTimestamp: "2026-07-08T16:42:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:42:49Z" name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r.18c05e25f8e0f268 namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: 6e179398-4720-4367-9267-bbc1929c42d0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:50Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 7a165d9e-b6f5-447d-be23-b27715bea795 kind: Event lastTimestamp: "2026-07-08T16:42:50Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:42:50Z" name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r.18c05e261256f741 namespace: kserve-ci-e2e-test resourceVersion: "14302" uid: 2aa33fd5-31d7-4e45-a9f6-883533ccab21 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r namespace: kserve-ci-e2e-test resourceVersion: "14271" uid: 7a165d9e-b6f5-447d-be23-b27715bea795 kind: Event lastTimestamp: "2026-07-08T16:42:50Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:43b18792ad3c8c70fe63e50dee47838dba7fd6b00a2dcfe15c76c1dafc2c7ad8" already present on machine metadata: creationTimestamp: "2026-07-08T16:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:42:50Z" name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r.18c05e2613f32fe3 namespace: kserve-ci-e2e-test resourceVersion: "14305" uid: 02e56438-9950-4de0-9a0e-bd9d1fd633a3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r namespace: kserve-ci-e2e-test resourceVersion: "14271" uid: 7a165d9e-b6f5-447d-be23-b27715bea795 kind: Event lastTimestamp: "2026-07-08T16:42:50Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T16:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:42:50Z" name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r.18c05e261739b20b namespace: kserve-ci-e2e-test resourceVersion: "14309" uid: 8e3314a2-149d-40ec-990b-fee0797bc5c0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r namespace: kserve-ci-e2e-test resourceVersion: "14271" uid: 7a165d9e-b6f5-447d-be23-b27715bea795 kind: Event lastTimestamp: "2026-07-08T16:42:50Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T16:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:42:50Z" name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r.18c05e2617a8cdd1 namespace: kserve-ci-e2e-test resourceVersion: "14310" uid: 385fda5b-1bc1-41c7-b349-51fd1e30b2c3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r namespace: kserve-ci-e2e-test resourceVersion: "14271" uid: 7a165d9e-b6f5-447d-be23-b27715bea795 kind: Event lastTimestamp: "2026-07-08T16:42:56Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1707" metadata: creationTimestamp: "2026-07-08T16:42:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:42:56Z" name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r.18c05e27754268da namespace: kserve-ci-e2e-test resourceVersion: "14384" uid: 929915f3-bf8c-41f5-96a2-95edd02f5314 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r namespace: kserve-ci-e2e-test resourceVersion: "14271" uid: 7a165d9e-b6f5-447d-be23-b27715bea795 kind: Event lastTimestamp: "2026-07-08T16:43:14Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1707" in 18.639s (18.639s including waiting). Image size: 1555648957 bytes.' metadata: creationTimestamp: "2026-07-08T16:43:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:14Z" name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r.18c05e2bcc44af43 namespace: kserve-ci-e2e-test resourceVersion: "14555" uid: 1aed88e7-b3c6-4d7c-b486-6e1348990f3d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r namespace: kserve-ci-e2e-test resourceVersion: "14271" uid: 7a165d9e-b6f5-447d-be23-b27715bea795 kind: Event lastTimestamp: "2026-07-08T16:43:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:43:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:22Z" name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r.18c05e2db687447e namespace: kserve-ci-e2e-test resourceVersion: "14628" uid: e1412bbc-4c00-423f-88e6-3e1114518c83 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r namespace: kserve-ci-e2e-test resourceVersion: "14271" uid: 7a165d9e-b6f5-447d-be23-b27715bea795 kind: Event lastTimestamp: "2026-07-08T16:43:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:43:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:22Z" name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r.18c05e2db7d7e65a namespace: kserve-ci-e2e-test resourceVersion: "14634" uid: 05ed7bc3-3c3b-4be6-bac2-fbc85c16c2ce reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r namespace: kserve-ci-e2e-test resourceVersion: "14271" uid: 7a165d9e-b6f5-447d-be23-b27715bea795 kind: Event lastTimestamp: "2026-07-08T16:43:22Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-08T16:43:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:22Z" name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r.18c05e2db7e55418 namespace: kserve-ci-e2e-test resourceVersion: "14636" uid: d12a2c7a-b47a-4de8-8e5c-84e2c55aec3c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r namespace: kserve-ci-e2e-test resourceVersion: "14271" uid: 7a165d9e-b6f5-447d-be23-b27715bea795 kind: Event lastTimestamp: "2026-07-08T16:43:25Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.464s (2.464s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-08T16:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:25Z" name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r.18c05e2e4ac80666 namespace: kserve-ci-e2e-test resourceVersion: "14661" uid: 87a7daa1-4a14-4fbc-ba40-00bad46bed66 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r namespace: kserve-ci-e2e-test resourceVersion: "14271" uid: 7a165d9e-b6f5-447d-be23-b27715bea795 kind: Event lastTimestamp: "2026-07-08T16:43:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:25Z" name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r.18c05e2e4f95977c namespace: kserve-ci-e2e-test resourceVersion: "14665" uid: de01d7da-019d-4313-b2c2-a953dfb066fe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r namespace: kserve-ci-e2e-test resourceVersion: "14271" uid: 7a165d9e-b6f5-447d-be23-b27715bea795 kind: Event lastTimestamp: "2026-07-08T16:43:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:25Z" name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r.18c05e2e502571a0 namespace: kserve-ci-e2e-test resourceVersion: "14666" uid: 9135b0df-cc9f-407a-b44e-61535a8fe2fd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T16:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r namespace: kserve-ci-e2e-test resourceVersion: "14271" uid: 7a165d9e-b6f5-447d-be23-b27715bea795 kind: Event lastTimestamp: "2026-07-08T16:45:22Z" message: 'Readiness probe failed: dial tcp 10.133.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:22Z" name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r.18c05e2e7c66b10c namespace: kserve-ci-e2e-test resourceVersion: "16049" uid: 0ef79b14-bb12-4a7c-88a2-4c1608b2cc49 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r namespace: kserve-ci-e2e-test resourceVersion: "14271" uid: 7a165d9e-b6f5-447d-be23-b27715bea795 kind: Event lastTimestamp: "2026-07-08T16:45:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:19Z" name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r.18c05e48d2d4c5da namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 9dac238d-8254-43df-a504-b5939126cf1b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r namespace: kserve-ci-e2e-test resourceVersion: "14271" uid: 7a165d9e-b6f5-447d-be23-b27715bea795 kind: Event lastTimestamp: "2026-07-08T16:45:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:19Z" name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r.18c05e48d2d54fda namespace: kserve-ci-e2e-test resourceVersion: "15912" uid: 1776ce62-9e84-4af1-93f2-d14dd517a138 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r namespace: kserve-ci-e2e-test resourceVersion: "14271" uid: 7a165d9e-b6f5-447d-be23-b27715bea795 kind: Event lastTimestamp: "2026-07-08T16:45:22Z" message: 'Readiness probe failed: Get "https://10.133.0.22:8643/healthz": dial tcp 10.133.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:45:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:22Z" name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r.18c05e497f105c98 namespace: kserve-ci-e2e-test resourceVersion: "16046" uid: eecbec5a-acb2-4158-aafc-2b933889539b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8 namespace: kserve-ci-e2e-test resourceVersion: "14262" uid: 9ee1f960-d1c9-4744-b933-d0dfd7b725ac kind: Event lastTimestamp: "2026-07-08T16:42:49Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-7f5c6b4bf8-87p9r' metadata: creationTimestamp: "2026-07-08T16:42:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:42:49Z" name: isvc-sklearn-graph-2-predictor-7f5c6b4bf8.18c05e25f86941b0 namespace: kserve-ci-e2e-test resourceVersion: "14270" uid: 39fb050f-cfa3-4f65-8d9b-fbcce72db812 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: 5aae900c-1bc1-4cf9-99fc-3acfe0e23afe kind: Event lastTimestamp: "2026-07-08T16:42:49Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-7f5c6b4bf8 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:42:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:42:49Z" name: isvc-sklearn-graph-2-predictor.18c05e25f52ab6d9 namespace: kserve-ci-e2e-test resourceVersion: "14263" uid: cc24c5a1-07a6-4c88-8eae-4a0352564579 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:43:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14261" uid: 9a6de848-f75a-490e-acef-75bf9dbeae2d kind: Event lastTimestamp: "2026-07-08T16:44:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:44:04Z" name: isvc-sklearn-graph-2-predictor.18c05e2973f9acc3 namespace: kserve-ci-e2e-test resourceVersion: "15009" uid: 7c584a06-0150-4ceb-b6ef-b3a84113a720 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:43:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14261" uid: 9a6de848-f75a-490e-acef-75bf9dbeae2d kind: Event lastTimestamp: "2026-07-08T16:44:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:44:04Z" name: isvc-sklearn-graph-2-predictor.18c05e29746d39df namespace: kserve-ci-e2e-test resourceVersion: "15011" uid: 17c8d61d-e104-4723-a1a8-cb764639d2fe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:43:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: 9a6de848-f75a-490e-acef-75bf9dbeae2d kind: Event lastTimestamp: "2026-07-08T16:45:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:45:04Z" name: isvc-sklearn-graph-2-predictor.18c05e33f0d0e9cd namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: f0a90b75-295f-41e3-b9c5-67e172dd02e1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:43:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: 9a6de848-f75a-490e-acef-75bf9dbeae2d kind: Event lastTimestamp: "2026-07-08T16:45:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:45:04Z" name: isvc-sklearn-graph-2-predictor.18c05e33f14ac9f6 namespace: kserve-ci-e2e-test resourceVersion: "15713" uid: 76bac016-c70c-4171-90bd-28fe3f18fac3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-08T16:42:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14252" uid: 4b6f5aae-751b-4d31-ab72-07d30f03c314 kind: Event lastTimestamp: "2026-07-08T16:44:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:42:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:44:42Z" name: isvc-sklearn-graph-2.18c05e25f56135e2 namespace: kserve-ci-e2e-test resourceVersion: "15518" uid: ba259d9b-0caa-4313-9193-e1608fffefb0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15513" uid: 4b6f5aae-751b-4d31-ab72-07d30f03c314 kind: Event lastTimestamp: "2026-07-08T16:44:42Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-08T16:44:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:44:42Z" name: isvc-sklearn-graph-2.18c05e40367c1892 namespace: kserve-ci-e2e-test resourceVersion: "15514" uid: 0483badc-0f05-4e21-beec-f557d93893eb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:42:49.392770Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8 namespace: kserve-ci-e2e-test resourceVersion: "14219" uid: 8e60c051-a083-4192-b735-3f490171c256 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-6kcb8 to ip-10-0-138-233.ec2.internal metadata: creationTimestamp: "2026-07-08T16:42:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:42:49Z" name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8.18c05e25eb4665e8 namespace: kserve-ci-e2e-test resourceVersion: "14224" uid: 6efefbe0-02ac-4b48-87d7-735119aa28e6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:49Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8 namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 8e60c051-a083-4192-b735-3f490171c256 kind: Event lastTimestamp: "2026-07-08T16:42:49Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T16:42:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:42:49Z" name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8.18c05e25f6a6a925 namespace: kserve-ci-e2e-test resourceVersion: "14268" uid: 276a721a-23ee-46b6-af44-e142eaf92b47 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:50Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8 namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: 8e60c051-a083-4192-b735-3f490171c256 kind: Event lastTimestamp: "2026-07-08T16:42:50Z" message: Add eth0 [10.133.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:42:50Z" name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8.18c05e26298e1d08 namespace: kserve-ci-e2e-test resourceVersion: "14318" uid: 940c1cbe-ffc1-401a-90e3-a1611080a63a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8 namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 8e60c051-a083-4192-b735-3f490171c256 kind: Event lastTimestamp: "2026-07-08T16:42:50Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:43b18792ad3c8c70fe63e50dee47838dba7fd6b00a2dcfe15c76c1dafc2c7ad8" already present on machine metadata: creationTimestamp: "2026-07-08T16:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:42:50Z" name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8.18c05e262b5292ac namespace: kserve-ci-e2e-test resourceVersion: "14320" uid: e524f1a0-6e3b-4108-afd5-6828e191cab9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8 namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 8e60c051-a083-4192-b735-3f490171c256 kind: Event lastTimestamp: "2026-07-08T16:42:50Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T16:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:42:50Z" name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8.18c05e262f684a4e namespace: kserve-ci-e2e-test resourceVersion: "14322" uid: e825f8bb-a411-4737-b5ef-5735e9b82b66 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8 namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 8e60c051-a083-4192-b735-3f490171c256 kind: Event lastTimestamp: "2026-07-08T16:42:50Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T16:42:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:42:50Z" name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8.18c05e262fd62503 namespace: kserve-ci-e2e-test resourceVersion: "14323" uid: 16fea429-4788-4d08-a775-1e124657a73c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8 namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 8e60c051-a083-4192-b735-3f490171c256 kind: Event lastTimestamp: "2026-07-08T16:42:56Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-08T16:42:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:42:56Z" name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8.18c05e277567fbe6 namespace: kserve-ci-e2e-test resourceVersion: "14386" uid: f0407882-1cd7-4219-b56d-3dbe83f498bc reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8 namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 8e60c051-a083-4192-b735-3f490171c256 kind: Event lastTimestamp: "2026-07-08T16:43:23Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 27.193s (27.193s including waiting). Image size: 1305547515 bytes.' metadata: creationTimestamp: "2026-07-08T16:43:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:23Z" name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8.18c05e2dca43668c namespace: kserve-ci-e2e-test resourceVersion: "14638" uid: 3fe50854-d445-4007-ae14-7346271d0424 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8 namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 8e60c051-a083-4192-b735-3f490171c256 kind: Event lastTimestamp: "2026-07-08T16:43:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:43:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:23Z" name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8.18c05e2dcd07b801 namespace: kserve-ci-e2e-test resourceVersion: "14640" uid: 44c25ecb-e601-4443-878f-a139d9f5fc2f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8 namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 8e60c051-a083-4192-b735-3f490171c256 kind: Event lastTimestamp: "2026-07-08T16:43:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:43:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:23Z" name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8.18c05e2dcd681813 namespace: kserve-ci-e2e-test resourceVersion: "14641" uid: 26dd31f7-3d5c-4f24-89a1-a7b89efeb8ed reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8 namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 8e60c051-a083-4192-b735-3f490171c256 kind: Event lastTimestamp: "2026-07-08T16:43:23Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-08T16:43:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:23Z" name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8.18c05e2dcd7548dc namespace: kserve-ci-e2e-test resourceVersion: "14642" uid: 758e5999-16e1-40a9-85ef-8eb6ed324887 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8 namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 8e60c051-a083-4192-b735-3f490171c256 kind: Event lastTimestamp: "2026-07-08T16:43:25Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.105s (2.105s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-08T16:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:25Z" name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8.18c05e2e4af3b39e namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: 123fddc2-9d60-4cbe-9eab-e5a20c93c79b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8 namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 8e60c051-a083-4192-b735-3f490171c256 kind: Event lastTimestamp: "2026-07-08T16:43:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:25Z" name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8.18c05e2e5064ddd4 namespace: kserve-ci-e2e-test resourceVersion: "14667" uid: 8ee220ce-23b4-4f64-9654-4db7bb5edfa0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8 namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 8e60c051-a083-4192-b735-3f490171c256 kind: Event lastTimestamp: "2026-07-08T16:43:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:25Z" name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8.18c05e2e517bbcd6 namespace: kserve-ci-e2e-test resourceVersion: "14671" uid: de936a56-e294-49ae-abaa-5fccdd646a09 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-08T16:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8 namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 8e60c051-a083-4192-b735-3f490171c256 kind: Event lastTimestamp: "2026-07-08T16:45:22Z" message: 'Readiness probe failed: dial tcp 10.133.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:22Z" name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8.18c05e2e7c7d5fd0 namespace: kserve-ci-e2e-test resourceVersion: "16048" uid: 3722efa0-a83a-4066-9456-cc6d3d59477b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8 namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 8e60c051-a083-4192-b735-3f490171c256 kind: Event lastTimestamp: "2026-07-08T16:45:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:19Z" name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8.18c05e48db26d161 namespace: kserve-ci-e2e-test resourceVersion: "15969" uid: 88be7411-c220-479c-aa85-b0486ae902bc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8 namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 8e60c051-a083-4192-b735-3f490171c256 kind: Event lastTimestamp: "2026-07-08T16:45:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:19Z" name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8.18c05e48db2d2b5b namespace: kserve-ci-e2e-test resourceVersion: "15972" uid: 688bfb1c-8537-40b4-a1ca-13b321fa8b46 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8 namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 8e60c051-a083-4192-b735-3f490171c256 kind: Event lastTimestamp: "2026-07-08T16:45:22Z" message: 'Readiness probe failed: Get "https://10.133.0.20:8643/healthz": dial tcp 10.133.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:45:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:22Z" name: isvc-xgboost-graph-predictor-555fc69d56-6kcb8.18c05e497f0f302b namespace: kserve-ci-e2e-test resourceVersion: "16045" uid: 25c9bdf5-5c5d-47f4-af7d-0df7aa263cf3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-233.ec2.internal source: component: kubelet host: ip-10-0-138-233.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14212" uid: ff04cfb8-cd20-4ecc-9b83-7003f7fc6667 kind: Event lastTimestamp: "2026-07-08T16:42:49Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-6kcb8' metadata: creationTimestamp: "2026-07-08T16:42:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:42:49Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c05e25ead926ad namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: 1dbe19b4-5faa-4342-a51b-e7e1a7909cff reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14210" uid: 80a0344b-3806-4713-b1c8-7c888e41c167 kind: Event lastTimestamp: "2026-07-08T16:42:49Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:42:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:42:49Z" name: isvc-xgboost-graph-predictor.18c05e25e7c391e1 namespace: kserve-ci-e2e-test resourceVersion: "14213" uid: 295e177d-a877-47e5-8140-6fcfddb846c1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:43:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14211" uid: e35cb9b5-5776-4d43-99b4-65686cd0f0f8 kind: Event lastTimestamp: "2026-07-08T16:44:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:44:04Z" name: isvc-xgboost-graph-predictor.18c05e29668ebfab namespace: kserve-ci-e2e-test resourceVersion: "15002" uid: 6017c3c0-211a-4cf0-8e2b-b217951dc2cf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:43:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14211" uid: e35cb9b5-5776-4d43-99b4-65686cd0f0f8 kind: Event lastTimestamp: "2026-07-08T16:44:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:44:04Z" name: isvc-xgboost-graph-predictor.18c05e2966fa982b namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: 0bc19ea3-e420-4492-9510-0f7d2bd558cb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:44:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14468" uid: e35cb9b5-5776-4d43-99b4-65686cd0f0f8 kind: Event lastTimestamp: "2026-07-08T16:45:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:45:04Z" name: isvc-xgboost-graph-predictor.18c05e3ae16a286c namespace: kserve-ci-e2e-test resourceVersion: "15708" uid: 10b70848-2992-4284-94d4-0ec55c64e1db reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:44:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14468" uid: e35cb9b5-5776-4d43-99b4-65686cd0f0f8 kind: Event lastTimestamp: "2026-07-08T16:45:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:44:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:45:04Z" name: isvc-xgboost-graph-predictor.18c05e3ae217ff23 namespace: kserve-ci-e2e-test resourceVersion: "15709" uid: e4b492f0-40c6-4fbd-ab88-ab69ac340b1a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-08T16:42:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14191" uid: 96743f89-a09d-4851-b69b-ec1d58601910 kind: Event lastTimestamp: "2026-07-08T16:44:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:42:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:44:32Z" name: isvc-xgboost-graph.18c05e25e7fe3beb namespace: kserve-ci-e2e-test resourceVersion: "15271" uid: 498efe04-b1a6-4665-a9a2-707491bda8f3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: 96743f89-a09d-4851-b69b-ec1d58601910 kind: Event lastTimestamp: "2026-07-08T16:44:32Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-08T16:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:44:32Z" name: isvc-xgboost-graph.18c05e3de0b81fd7 namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: a63a0449-c9d8-4376-96b5-0a5e9b7a4f6c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:45:09.035272Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-bdf5fb5f7-bnnvv namespace: kserve-ci-e2e-test resourceVersion: "15756" uid: 3f3a74be-312a-403c-8f10-89053f52c915 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-bdf5fb5f7-bnnvv to ip-10-0-132-182.ec2.internal metadata: creationTimestamp: "2026-07-08T16:45:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:45:09Z" name: model-chainer-bdf5fb5f7-bnnvv.18c05e466e9de359 namespace: kserve-ci-e2e-test resourceVersion: "15761" uid: 5e7cde9f-be35-462f-bbcb-4d46071266d9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:09Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-bdf5fb5f7-bnnvv namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: 3f3a74be-312a-403c-8f10-89053f52c915 kind: Event lastTimestamp: "2026-07-08T16:45:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T16:45:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:09Z" name: model-chainer-bdf5fb5f7-bnnvv.18c05e467740ffea namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: 76671040-6c5d-4596-9a00-8e4227649753 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:10Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-bdf5fb5f7-bnnvv namespace: kserve-ci-e2e-test resourceVersion: "15765" uid: 3f3a74be-312a-403c-8f10-89053f52c915 kind: Event lastTimestamp: "2026-07-08T16:45:10Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:45:10Z" name: model-chainer-bdf5fb5f7-bnnvv.18c05e46ac21c3e2 namespace: kserve-ci-e2e-test resourceVersion: "15790" uid: b7321bc2-fbcb-4dd6-a567-6495ea6d81d9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-bdf5fb5f7-bnnvv namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: 3f3a74be-312a-403c-8f10-89053f52c915 kind: Event lastTimestamp: "2026-07-08T16:45:10Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:86bfaabcc176433317a84dc02e835b9657d32290b963c2e4ae8312aa72bd9d64" already present on machine metadata: creationTimestamp: "2026-07-08T16:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:10Z" name: model-chainer-bdf5fb5f7-bnnvv.18c05e46ad717d02 namespace: kserve-ci-e2e-test resourceVersion: "15792" uid: 451c8cce-0e78-460b-b239-d8828f9f3078 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-bdf5fb5f7-bnnvv namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: 3f3a74be-312a-403c-8f10-89053f52c915 kind: Event lastTimestamp: "2026-07-08T16:45:10Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-08T16:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:10Z" name: model-chainer-bdf5fb5f7-bnnvv.18c05e46b02feb4a namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: 16ae0c61-06f5-4775-bce0-7dd637dafa91 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-bdf5fb5f7-bnnvv namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: 3f3a74be-312a-403c-8f10-89053f52c915 kind: Event lastTimestamp: "2026-07-08T16:45:10Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-08T16:45:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:10Z" name: model-chainer-bdf5fb5f7-bnnvv.18c05e46b09097d0 namespace: kserve-ci-e2e-test resourceVersion: "15795" uid: 14374af8-8427-4607-b563-48c375c6b5b3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-bdf5fb5f7-bnnvv namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: 3f3a74be-312a-403c-8f10-89053f52c915 kind: Event lastTimestamp: "2026-07-08T16:45:19Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:19Z" name: model-chainer-bdf5fb5f7-bnnvv.18c05e48ca054341 namespace: kserve-ci-e2e-test resourceVersion: "15882" uid: 85467e5c-16fe-486e-9f62-db5a408e83a3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:45:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-bdf5fb5f7-bnnvv namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: 3f3a74be-312a-403c-8f10-89053f52c915 kind: Event lastTimestamp: "2026-07-08T16:45:46Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T16:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:46Z" name: model-chainer-bdf5fb5f7-bnnvv.18c05e4972e4723c namespace: kserve-ci-e2e-test resourceVersion: "16315" uid: c4d3f25f-90f2-4665-b4f3-6c7879863109 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-bdf5fb5f7 namespace: kserve-ci-e2e-test resourceVersion: "15753" uid: 7aaaf6ee-315a-4802-83b8-4ab98eabe0f0 kind: Event lastTimestamp: "2026-07-08T16:45:09Z" message: 'Created pod: model-chainer-bdf5fb5f7-bnnvv' metadata: creationTimestamp: "2026-07-08T16:45:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:45:09Z" name: model-chainer-bdf5fb5f7.18c05e466e2dbdb8 namespace: kserve-ci-e2e-test resourceVersion: "15757" uid: ed0619d3-1537-4fb1-88ec-8d022041f74c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15751" uid: d0cfeefa-074b-4ce6-a3fe-df012c1ee378 kind: Event lastTimestamp: "2026-07-08T16:45:09Z" message: Scaled up replica set model-chainer-bdf5fb5f7 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:45:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:45:09Z" name: model-chainer.18c05e466d38e443 namespace: kserve-ci-e2e-test resourceVersion: "15754" uid: 47801d22-b281-482a-95e2-e941aab5fcd0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:09Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15768" uid: 4eb0a3c1-1c69-4446-896e-444b3f45b7ce kind: Event lastTimestamp: "2026-07-08T16:45:09Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-08T16:45:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:45:09Z" name: model-chainer.18c05e4672dcf594 namespace: kserve-ci-e2e-test resourceVersion: "15770" uid: 25a1d741-45c4-4951-b1ff-9b5d730f0e61 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:55:34.209143Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-3816e-6d744db655-797vb namespace: kserve-ci-e2e-test resourceVersion: "21041" uid: 8c00ae3b-6cda-471d-9fcf-c8245a7ae654 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-3816e-6d744db655-797vb to ip-10-0-132-182.ec2.internal metadata: creationTimestamp: "2026-07-08T16:55:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:55:34Z" name: sequence-graph-3816e-6d744db655-797vb.18c05ed7fde212ed namespace: kserve-ci-e2e-test resourceVersion: "21046" uid: 8813592c-b437-4d94-8057-ce41bbfa8d56 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:34Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-3816e-6d744db655-797vb namespace: kserve-ci-e2e-test resourceVersion: "21050" uid: 8c00ae3b-6cda-471d-9fcf-c8245a7ae654 kind: Event lastTimestamp: "2026-07-08T16:55:34Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:55:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:55:34Z" name: sequence-graph-3816e-6d744db655-797vb.18c05ed818bd2671 namespace: kserve-ci-e2e-test resourceVersion: "21069" uid: e8bcf7ba-18f1-4d5c-ab50-d1c905de21cf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3816e} kind: Pod name: sequence-graph-3816e-6d744db655-797vb namespace: kserve-ci-e2e-test resourceVersion: "21043" uid: 8c00ae3b-6cda-471d-9fcf-c8245a7ae654 kind: Event lastTimestamp: "2026-07-08T16:55:34Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:86bfaabcc176433317a84dc02e835b9657d32290b963c2e4ae8312aa72bd9d64" already present on machine metadata: creationTimestamp: "2026-07-08T16:55:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:34Z" name: sequence-graph-3816e-6d744db655-797vb.18c05ed81a3582f4 namespace: kserve-ci-e2e-test resourceVersion: "21071" uid: b2efcdef-dc6b-4629-9a28-c41209d7fe0b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3816e} kind: Pod name: sequence-graph-3816e-6d744db655-797vb namespace: kserve-ci-e2e-test resourceVersion: "21043" uid: 8c00ae3b-6cda-471d-9fcf-c8245a7ae654 kind: Event lastTimestamp: "2026-07-08T16:55:34Z" message: 'Created container: sequence-graph-3816e' metadata: creationTimestamp: "2026-07-08T16:55:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:34Z" name: sequence-graph-3816e-6d744db655-797vb.18c05ed81cd9626d namespace: kserve-ci-e2e-test resourceVersion: "21072" uid: 2127627f-bba1-4076-a450-3735c74aea1e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3816e} kind: Pod name: sequence-graph-3816e-6d744db655-797vb namespace: kserve-ci-e2e-test resourceVersion: "21043" uid: 8c00ae3b-6cda-471d-9fcf-c8245a7ae654 kind: Event lastTimestamp: "2026-07-08T16:55:34Z" message: Started container sequence-graph-3816e metadata: creationTimestamp: "2026-07-08T16:55:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:34Z" name: sequence-graph-3816e-6d744db655-797vb.18c05ed81d86e3b5 namespace: kserve-ci-e2e-test resourceVersion: "21073" uid: 869ffcc0-96b1-4d71-8f40-0f8c4934164f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3816e} kind: Pod name: sequence-graph-3816e-6d744db655-797vb namespace: kserve-ci-e2e-test resourceVersion: "21043" uid: 8c00ae3b-6cda-471d-9fcf-c8245a7ae654 kind: Event lastTimestamp: "2026-07-08T16:55:44Z" message: Stopping container sequence-graph-3816e metadata: creationTimestamp: "2026-07-08T16:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:44Z" name: sequence-graph-3816e-6d744db655-797vb.18c05eda5690a3fa namespace: kserve-ci-e2e-test resourceVersion: "21172" uid: 7982d4cf-287f-434a-bf29-0cba6742de7a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:55:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3816e} kind: Pod name: sequence-graph-3816e-6d744db655-797vb namespace: kserve-ci-e2e-test resourceVersion: "21043" uid: 8c00ae3b-6cda-471d-9fcf-c8245a7ae654 kind: Event lastTimestamp: "2026-07-08T16:56:11Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T16:55:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:56:11Z" name: sequence-graph-3816e-6d744db655-797vb.18c05edae4600bc7 namespace: kserve-ci-e2e-test resourceVersion: "21579" uid: 6b7a14ce-2f5b-4ef9-be0b-952601c3814a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-3816e-6d744db655 namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: c615e025-6d6a-45c6-a355-0432eeafd900 kind: Event lastTimestamp: "2026-07-08T16:55:34Z" message: 'Created pod: sequence-graph-3816e-6d744db655-797vb' metadata: creationTimestamp: "2026-07-08T16:55:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:34Z" name: sequence-graph-3816e-6d744db655.18c05ed7fd449e20 namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: ff74181f-5df9-4b5c-833b-bab5439c9bfd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-3816e namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: d9e0358d-0263-4cb4-9216-e7b13e42dd49 kind: Event lastTimestamp: "2026-07-08T16:55:34Z" message: Scaled up replica set sequence-graph-3816e-6d744db655 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:55:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:34Z" name: sequence-graph-3816e.18c05ed7fc2d8e56 namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: 68f59c82-ac61-4dec-afd9-d6f1e786ce80 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:34Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-3816e namespace: kserve-ci-e2e-test resourceVersion: "21055" uid: af99056f-64ac-46b7-9cc8-b40e45150bc9 kind: Event lastTimestamp: "2026-07-08T16:55:34Z" message: InferenceGraph [sequence-graph-3816e] is Ready metadata: creationTimestamp: "2026-07-08T16:55:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:55:34Z" name: sequence-graph-3816e.18c05ed802a39ca6 namespace: kserve-ci-e2e-test resourceVersion: "21056" uid: 9eed8603-15d2-4dd7-89b3-b46e489ba4fc reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:56:44.547055Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f6dd6-cbdd594c5-jgpqf namespace: kserve-ci-e2e-test resourceVersion: "21873" uid: c532c61d-3d07-47d9-bad2-19801079abab kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-f6dd6-cbdd594c5-jgpqf to ip-10-0-132-182.ec2.internal metadata: creationTimestamp: "2026-07-08T16:56:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:56:44Z" name: sequence-graph-f6dd6-cbdd594c5-jgpqf.18c05ee85e597454 namespace: kserve-ci-e2e-test resourceVersion: "21878" uid: 3340f3de-5502-4d5a-ad76-52b61304e392 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:44Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f6dd6-cbdd594c5-jgpqf namespace: kserve-ci-e2e-test resourceVersion: "21874" uid: c532c61d-3d07-47d9-bad2-19801079abab kind: Event lastTimestamp: "2026-07-08T16:56:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-f6dd6-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T16:56:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:56:44Z" name: sequence-graph-f6dd6-cbdd594c5-jgpqf.18c05ee86658fd61 namespace: kserve-ci-e2e-test resourceVersion: "21891" uid: 62cf4d88-df49-478f-ae43-25b22622e6ac reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:45Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f6dd6-cbdd594c5-jgpqf namespace: kserve-ci-e2e-test resourceVersion: "21882" uid: c532c61d-3d07-47d9-bad2-19801079abab kind: Event lastTimestamp: "2026-07-08T16:56:45Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:56:45Z" name: sequence-graph-f6dd6-cbdd594c5-jgpqf.18c05ee89bb2becd namespace: kserve-ci-e2e-test resourceVersion: "21898" uid: 05f1c172-08c2-467c-8cf6-e07ca8b0b44d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f6dd6} kind: Pod name: sequence-graph-f6dd6-cbdd594c5-jgpqf namespace: kserve-ci-e2e-test resourceVersion: "21874" uid: c532c61d-3d07-47d9-bad2-19801079abab kind: Event lastTimestamp: "2026-07-08T16:56:45Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:86bfaabcc176433317a84dc02e835b9657d32290b963c2e4ae8312aa72bd9d64" already present on machine metadata: creationTimestamp: "2026-07-08T16:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:56:45Z" name: sequence-graph-f6dd6-cbdd594c5-jgpqf.18c05ee89d947452 namespace: kserve-ci-e2e-test resourceVersion: "21900" uid: 5d69b637-5195-4722-9df3-23b9c03b8f4c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f6dd6} kind: Pod name: sequence-graph-f6dd6-cbdd594c5-jgpqf namespace: kserve-ci-e2e-test resourceVersion: "21874" uid: c532c61d-3d07-47d9-bad2-19801079abab kind: Event lastTimestamp: "2026-07-08T16:56:45Z" message: 'Created container: sequence-graph-f6dd6' metadata: creationTimestamp: "2026-07-08T16:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:56:45Z" name: sequence-graph-f6dd6-cbdd594c5-jgpqf.18c05ee8a079de0a namespace: kserve-ci-e2e-test resourceVersion: "21903" uid: 01d2b1a0-4aec-4167-915c-93ced45acc63 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f6dd6} kind: Pod name: sequence-graph-f6dd6-cbdd594c5-jgpqf namespace: kserve-ci-e2e-test resourceVersion: "21874" uid: c532c61d-3d07-47d9-bad2-19801079abab kind: Event lastTimestamp: "2026-07-08T16:56:45Z" message: Started container sequence-graph-f6dd6 metadata: creationTimestamp: "2026-07-08T16:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:56:45Z" name: sequence-graph-f6dd6-cbdd594c5-jgpqf.18c05ee8a109d69d namespace: kserve-ci-e2e-test resourceVersion: "21904" uid: 30aae1e7-6173-4a80-957e-c7b14a5d0a74 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f6dd6} kind: Pod name: sequence-graph-f6dd6-cbdd594c5-jgpqf namespace: kserve-ci-e2e-test resourceVersion: "21874" uid: c532c61d-3d07-47d9-bad2-19801079abab kind: Event lastTimestamp: "2026-07-08T17:04:59Z" message: Stopping container sequence-graph-f6dd6 metadata: creationTimestamp: "2026-07-08T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:59Z" name: sequence-graph-f6dd6-cbdd594c5-jgpqf.18c05f5b8d743912 namespace: kserve-ci-e2e-test resourceVersion: "25744" uid: f082353c-1d15-4bb5-80ee-0440ab83ea39 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T17:05:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f6dd6} kind: Pod name: sequence-graph-f6dd6-cbdd594c5-jgpqf namespace: kserve-ci-e2e-test resourceVersion: "21874" uid: c532c61d-3d07-47d9-bad2-19801079abab kind: Event lastTimestamp: "2026-07-08T17:05:26Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T17:05:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:26Z" name: sequence-graph-f6dd6-cbdd594c5-jgpqf.18c05f5c300e8dde namespace: kserve-ci-e2e-test resourceVersion: "26146" uid: 4636ed54-e79d-46f1-b04a-01cc3a82117f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-f6dd6-cbdd594c5 namespace: kserve-ci-e2e-test resourceVersion: "21870" uid: fc092711-2a85-40d0-9a04-d71799a0b9d4 kind: Event lastTimestamp: "2026-07-08T16:56:44Z" message: 'Created pod: sequence-graph-f6dd6-cbdd594c5-jgpqf' metadata: creationTimestamp: "2026-07-08T16:56:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:56:44Z" name: sequence-graph-f6dd6-cbdd594c5.18c05ee85dc80302 namespace: kserve-ci-e2e-test resourceVersion: "21875" uid: 293b8ab9-60de-4e71-bdf0-df50b0f8726e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-f6dd6 namespace: kserve-ci-e2e-test resourceVersion: "21866" uid: 1ade6cba-e51a-481a-bc25-fe354905ba1c kind: Event lastTimestamp: "2026-07-08T16:56:44Z" message: Scaled up replica set sequence-graph-f6dd6-cbdd594c5 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:56:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:56:44Z" name: sequence-graph-f6dd6.18c05ee85ced86fa namespace: kserve-ci-e2e-test resourceVersion: "21871" uid: ee751071-ffe8-4c31-be17-76cd08e967c2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:44Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-f6dd6 namespace: kserve-ci-e2e-test resourceVersion: "21887" uid: 2e2abc44-c69b-4792-a9b6-6716b7bd202c kind: Event lastTimestamp: "2026-07-08T16:56:44Z" message: InferenceGraph [sequence-graph-f6dd6] is Ready metadata: creationTimestamp: "2026-07-08T16:56:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:56:44Z" name: sequence-graph-f6dd6.18c05ee862c869d0 namespace: kserve-ci-e2e-test resourceVersion: "21889" uid: 48c64387-6ba7-41b8-ad48-e81b416ca54f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:56:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f6dd6 namespace: kserve-ci-e2e-test resourceVersion: "21868" uid: b350b4dc-da98-4ea2-bbde-2136dae86f51 kind: Event lastTimestamp: "2026-07-08T16:57:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:56:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:57:14Z" name: sequence-graph-f6dd6.18c05eebdb5dff86 namespace: kserve-ci-e2e-test resourceVersion: "22120" uid: 89e955c9-b17f-4cc7-a14e-ca3794dda179 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:56:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f6dd6 namespace: kserve-ci-e2e-test resourceVersion: "21868" uid: b350b4dc-da98-4ea2-bbde-2136dae86f51 kind: Event lastTimestamp: "2026-07-08T16:57:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:56:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:57:14Z" name: sequence-graph-f6dd6.18c05eebdbc1f6e3 namespace: kserve-ci-e2e-test resourceVersion: "22121" uid: 32f1e0d8-0b7e-4882-98ca-4929cdbe80b6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:46:19.375805Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-fabf2-5f6647c4bf-kcmn7 namespace: kserve-ci-e2e-test resourceVersion: "16591" uid: a14cd999-092f-472b-97c7-563be16c2261 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-fabf2-5f6647c4bf-kcmn7 to ip-10-0-132-182.ec2.internal metadata: creationTimestamp: "2026-07-08T16:46:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:46:19Z" name: sequence-graph-fabf2-5f6647c4bf-kcmn7.18c05e56cf3d4075 namespace: kserve-ci-e2e-test resourceVersion: "16596" uid: 893f6d41-64a8-444a-a200-3c491e74a55d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:19Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-fabf2-5f6647c4bf-kcmn7 namespace: kserve-ci-e2e-test resourceVersion: "16592" uid: a14cd999-092f-472b-97c7-563be16c2261 kind: Event lastTimestamp: "2026-07-08T16:46:19Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-fabf2-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T16:46:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:19Z" name: sequence-graph-fabf2-5f6647c4bf-kcmn7.18c05e56d9527cce namespace: kserve-ci-e2e-test resourceVersion: "16614" uid: 317d0dea-830a-4295-8386-bc43b33a3141 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:20Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-fabf2-5f6647c4bf-kcmn7 namespace: kserve-ci-e2e-test resourceVersion: "16600" uid: a14cd999-092f-472b-97c7-563be16c2261 kind: Event lastTimestamp: "2026-07-08T16:46:20Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:46:20Z" name: sequence-graph-fabf2-5f6647c4bf-kcmn7.18c05e570c8d28fc namespace: kserve-ci-e2e-test resourceVersion: "16623" uid: a64c0072-4cac-4752-979e-f7462e2efea1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fabf2} kind: Pod name: sequence-graph-fabf2-5f6647c4bf-kcmn7 namespace: kserve-ci-e2e-test resourceVersion: "16592" uid: a14cd999-092f-472b-97c7-563be16c2261 kind: Event lastTimestamp: "2026-07-08T16:46:20Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:86bfaabcc176433317a84dc02e835b9657d32290b963c2e4ae8312aa72bd9d64" already present on machine metadata: creationTimestamp: "2026-07-08T16:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:20Z" name: sequence-graph-fabf2-5f6647c4bf-kcmn7.18c05e570de5e0ed namespace: kserve-ci-e2e-test resourceVersion: "16625" uid: c442b03b-6c48-47b7-9707-54f096910468 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fabf2} kind: Pod name: sequence-graph-fabf2-5f6647c4bf-kcmn7 namespace: kserve-ci-e2e-test resourceVersion: "16592" uid: a14cd999-092f-472b-97c7-563be16c2261 kind: Event lastTimestamp: "2026-07-08T16:46:20Z" message: 'Created container: sequence-graph-fabf2' metadata: creationTimestamp: "2026-07-08T16:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:20Z" name: sequence-graph-fabf2-5f6647c4bf-kcmn7.18c05e5711a07474 namespace: kserve-ci-e2e-test resourceVersion: "16626" uid: 578cc855-b4b2-474c-aa3b-d8fc1897d7de reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fabf2} kind: Pod name: sequence-graph-fabf2-5f6647c4bf-kcmn7 namespace: kserve-ci-e2e-test resourceVersion: "16592" uid: a14cd999-092f-472b-97c7-563be16c2261 kind: Event lastTimestamp: "2026-07-08T16:46:20Z" message: Started container sequence-graph-fabf2 metadata: creationTimestamp: "2026-07-08T16:46:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:20Z" name: sequence-graph-fabf2-5f6647c4bf-kcmn7.18c05e57120b10be namespace: kserve-ci-e2e-test resourceVersion: "16627" uid: 451614de-8fff-40ae-bbe3-651202d766a9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fabf2} kind: Pod name: sequence-graph-fabf2-5f6647c4bf-kcmn7 namespace: kserve-ci-e2e-test resourceVersion: "16592" uid: a14cd999-092f-472b-97c7-563be16c2261 kind: Event lastTimestamp: "2026-07-08T16:54:33Z" message: Stopping container sequence-graph-fabf2 metadata: creationTimestamp: "2026-07-08T16:54:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:33Z" name: sequence-graph-fabf2-5f6647c4bf-kcmn7.18c05ec9f69a10f0 namespace: kserve-ci-e2e-test resourceVersion: "20201" uid: c7f302bc-31ce-46dc-860a-64543e6543bc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:54:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fabf2} kind: Pod name: sequence-graph-fabf2-5f6647c4bf-kcmn7 namespace: kserve-ci-e2e-test resourceVersion: "16592" uid: a14cd999-092f-472b-97c7-563be16c2261 kind: Event lastTimestamp: "2026-07-08T16:55:02Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T16:54:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:02Z" name: sequence-graph-fabf2-5f6647c4bf-kcmn7.18c05ecabd4d5116 namespace: kserve-ci-e2e-test resourceVersion: "20614" uid: 10a5dcef-cbad-4aa7-8f52-ce90decec0db reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-fabf2-5f6647c4bf namespace: kserve-ci-e2e-test resourceVersion: "16588" uid: 2f79557b-c719-41ff-a990-7a3d96f8e44a kind: Event lastTimestamp: "2026-07-08T16:46:19Z" message: 'Created pod: sequence-graph-fabf2-5f6647c4bf-kcmn7' metadata: creationTimestamp: "2026-07-08T16:46:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:19Z" name: sequence-graph-fabf2-5f6647c4bf.18c05e56ced91a7b namespace: kserve-ci-e2e-test resourceVersion: "16593" uid: 671c340f-2409-4ff7-830f-c9e5f31fcc03 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-fabf2 namespace: kserve-ci-e2e-test resourceVersion: "16585" uid: 7b03c8d5-1b8c-4acb-b0c0-33ce0d032c6c kind: Event lastTimestamp: "2026-07-08T16:46:19Z" message: Scaled up replica set sequence-graph-fabf2-5f6647c4bf from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:46:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:19Z" name: sequence-graph-fabf2.18c05e56cde36eb6 namespace: kserve-ci-e2e-test resourceVersion: "16589" uid: 80f1352a-b765-48f7-86af-8889d4a55665 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-fabf2 namespace: kserve-ci-e2e-test resourceVersion: "16607" uid: aa84163a-2aa3-4ef5-ab84-5545fe5fbecb kind: Event lastTimestamp: "2026-07-08T16:46:19Z" message: InferenceGraph [sequence-graph-fabf2] is Ready metadata: creationTimestamp: "2026-07-08T16:46:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:46:19Z" name: sequence-graph-fabf2.18c05e56d3c1c9e4 namespace: kserve-ci-e2e-test resourceVersion: "16609" uid: 82f93403-b7d5-403b-892f-19305138307c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-fabf2 namespace: kserve-ci-e2e-test resourceVersion: "16587" uid: 76aa602c-5e90-40b0-94f9-cfc25b305f70 kind: Event lastTimestamp: "2026-07-08T16:46:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:46:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:34Z" name: sequence-graph-fabf2.18c05e5a4c6fd9cd namespace: kserve-ci-e2e-test resourceVersion: "16729" uid: b80a7109-724d-4c1c-9b75-888d5319d09d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-fabf2 namespace: kserve-ci-e2e-test resourceVersion: "16587" uid: 76aa602c-5e90-40b0-94f9-cfc25b305f70 kind: Event lastTimestamp: "2026-07-08T16:46:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:46:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:34Z" name: sequence-graph-fabf2.18c05e5a4cd697f8 namespace: kserve-ci-e2e-test resourceVersion: "16730" uid: ee81fd3e-ef07-4994-b363-6813a0a711aa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-fabf2 namespace: kserve-ci-e2e-test resourceVersion: "16728" uid: 76aa602c-5e90-40b0-94f9-cfc25b305f70 kind: Event lastTimestamp: "2026-07-08T16:46:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:49Z" name: sequence-graph-fabf2.18c05e5dcc487b1c namespace: kserve-ci-e2e-test resourceVersion: "16836" uid: 4aee607b-ebad-4360-bbcd-d6621546b84b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-fabf2 namespace: kserve-ci-e2e-test resourceVersion: "16728" uid: 76aa602c-5e90-40b0-94f9-cfc25b305f70 kind: Event lastTimestamp: "2026-07-08T16:46:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:46:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:49Z" name: sequence-graph-fabf2.18c05e5dccbd10fe namespace: kserve-ci-e2e-test resourceVersion: "16837" uid: 492712b0-08c6-4ae8-b6a1-5ec71a871036 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-08T17:05:19.558060Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-78c4a-56cf4ffc7b-l8qvw namespace: kserve-ci-e2e-test resourceVersion: "26062" uid: 3f515122-b53d-41f8-aea1-ed4c75c91d74 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-78c4a-56cf4ffc7b-l8qvw to ip-10-0-132-182.ec2.internal metadata: creationTimestamp: "2026-07-08T17:05:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T17:05:19Z" name: splitter-graph-78c4a-56cf4ffc7b-l8qvw.18c05f604765c4cb namespace: kserve-ci-e2e-test resourceVersion: "26067" uid: 4f6d511b-da1a-4e7d-a19e-3d3cf96ea5ab reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:19Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-78c4a-56cf4ffc7b-l8qvw namespace: kserve-ci-e2e-test resourceVersion: "26075" uid: 3f515122-b53d-41f8-aea1-ed4c75c91d74 kind: Event lastTimestamp: "2026-07-08T17:05:19Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T17:05:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T17:05:19Z" name: splitter-graph-78c4a-56cf4ffc7b-l8qvw.18c05f606119c316 namespace: kserve-ci-e2e-test resourceVersion: "26084" uid: 9584f776-ce19-42a2-a3d9-27abbabd6ea9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-78c4a} kind: Pod name: splitter-graph-78c4a-56cf4ffc7b-l8qvw namespace: kserve-ci-e2e-test resourceVersion: "26063" uid: 3f515122-b53d-41f8-aea1-ed4c75c91d74 kind: Event lastTimestamp: "2026-07-08T17:05:20Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:86bfaabcc176433317a84dc02e835b9657d32290b963c2e4ae8312aa72bd9d64" already present on machine metadata: creationTimestamp: "2026-07-08T17:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:20Z" name: splitter-graph-78c4a-56cf4ffc7b-l8qvw.18c05f606265d472 namespace: kserve-ci-e2e-test resourceVersion: "26086" uid: 56ee7bbd-4ab6-4ac9-b91b-eec268294254 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-78c4a} kind: Pod name: splitter-graph-78c4a-56cf4ffc7b-l8qvw namespace: kserve-ci-e2e-test resourceVersion: "26063" uid: 3f515122-b53d-41f8-aea1-ed4c75c91d74 kind: Event lastTimestamp: "2026-07-08T17:05:20Z" message: 'Created container: splitter-graph-78c4a' metadata: creationTimestamp: "2026-07-08T17:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:20Z" name: splitter-graph-78c4a-56cf4ffc7b-l8qvw.18c05f6065421b7b namespace: kserve-ci-e2e-test resourceVersion: "26088" uid: bf941ca6-657a-43b9-bb64-55789654c07c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-78c4a} kind: Pod name: splitter-graph-78c4a-56cf4ffc7b-l8qvw namespace: kserve-ci-e2e-test resourceVersion: "26063" uid: 3f515122-b53d-41f8-aea1-ed4c75c91d74 kind: Event lastTimestamp: "2026-07-08T17:05:20Z" message: Started container splitter-graph-78c4a metadata: creationTimestamp: "2026-07-08T17:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:20Z" name: splitter-graph-78c4a-56cf4ffc7b-l8qvw.18c05f6065b40bbf namespace: kserve-ci-e2e-test resourceVersion: "26089" uid: cafa1bf7-692d-488d-9bd2-5f276bf55629 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-78c4a} kind: Pod name: splitter-graph-78c4a-56cf4ffc7b-l8qvw namespace: kserve-ci-e2e-test resourceVersion: "26063" uid: 3f515122-b53d-41f8-aea1-ed4c75c91d74 kind: Event lastTimestamp: "2026-07-08T17:05:29Z" message: Stopping container splitter-graph-78c4a metadata: creationTimestamp: "2026-07-08T17:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:29Z" name: splitter-graph-78c4a-56cf4ffc7b-l8qvw.18c05f62a0d6e32a namespace: kserve-ci-e2e-test resourceVersion: "26187" uid: ceadc075-a368-4850-ad24-d5d985f12064 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T17:05:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-78c4a} kind: Pod name: splitter-graph-78c4a-56cf4ffc7b-l8qvw namespace: kserve-ci-e2e-test resourceVersion: "26063" uid: 3f515122-b53d-41f8-aea1-ed4c75c91d74 kind: Event lastTimestamp: "2026-07-08T17:05:56Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T17:05:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:56Z" name: splitter-graph-78c4a-56cf4ffc7b-l8qvw.18c05f62fc2f2eb2 namespace: kserve-ci-e2e-test resourceVersion: "26544" uid: e50bfa57-ebf4-4ea7-be61-4d82c09ec65b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-78c4a-56cf4ffc7b namespace: kserve-ci-e2e-test resourceVersion: "26059" uid: 59107f3c-7156-4ea8-8f6a-2a35db66a1a1 kind: Event lastTimestamp: "2026-07-08T17:05:19Z" message: 'Created pod: splitter-graph-78c4a-56cf4ffc7b-l8qvw' metadata: creationTimestamp: "2026-07-08T17:05:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:05:19Z" name: splitter-graph-78c4a-56cf4ffc7b.18c05f604700ba7d namespace: kserve-ci-e2e-test resourceVersion: "26064" uid: 8e1e86c5-8c95-461b-b3d2-91d36d00c9ae reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-78c4a namespace: kserve-ci-e2e-test resourceVersion: "26056" uid: 9186172e-594d-4df7-a57a-c52327d61902 kind: Event lastTimestamp: "2026-07-08T17:05:19Z" message: Scaled up replica set splitter-graph-78c4a-56cf4ffc7b from 0 to 1 metadata: creationTimestamp: "2026-07-08T17:05:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:05:19Z" name: splitter-graph-78c4a.18c05f60462bf2fd namespace: kserve-ci-e2e-test resourceVersion: "26060" uid: 3399c311-bea0-4c0c-baea-b0c3498948a3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-78c4a namespace: kserve-ci-e2e-test resourceVersion: "26079" uid: 8c7d818e-691b-4eaa-8f7e-a1ab855ca29c kind: Event lastTimestamp: "2026-07-08T17:05:19Z" message: InferenceGraph [splitter-graph-78c4a] is Ready metadata: creationTimestamp: "2026-07-08T17:05:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:05:19Z" name: splitter-graph-78c4a.18c05f604c30343c namespace: kserve-ci-e2e-test resourceVersion: "26081" uid: b92677ac-e375-4700-9be4-c89310e026a5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T17:06:29.903845Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-91fdc-d7f8d8f4c-q42gc namespace: kserve-ci-e2e-test resourceVersion: "26881" uid: e352b067-8f1d-4604-b3ad-4731c9c07ef5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-91fdc-d7f8d8f4c-q42gc to ip-10-0-132-182.ec2.internal metadata: creationTimestamp: "2026-07-08T17:06:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T17:06:29Z" name: splitter-graph-91fdc-d7f8d8f4c-q42gc.18c05f70a8554087 namespace: kserve-ci-e2e-test resourceVersion: "26886" uid: 20911550-bf3d-44c3-bf3b-247ae06242a7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:06:30Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-91fdc-d7f8d8f4c-q42gc namespace: kserve-ci-e2e-test resourceVersion: "26883" uid: e352b067-8f1d-4604-b3ad-4731c9c07ef5 kind: Event lastTimestamp: "2026-07-08T17:06:30Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-91fdc-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T17:06:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:06:30Z" name: splitter-graph-91fdc-d7f8d8f4c-q42gc.18c05f70b19f6184 namespace: kserve-ci-e2e-test resourceVersion: "26900" uid: 77af4a8b-ed65-4a80-a61e-5e1f632f41e5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:06:30Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-91fdc-d7f8d8f4c-q42gc namespace: kserve-ci-e2e-test resourceVersion: "26890" uid: e352b067-8f1d-4604-b3ad-4731c9c07ef5 kind: Event lastTimestamp: "2026-07-08T17:06:30Z" message: Add eth0 [10.132.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T17:06:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T17:06:30Z" name: splitter-graph-91fdc-d7f8d8f4c-q42gc.18c05f70e598d026 namespace: kserve-ci-e2e-test resourceVersion: "26908" uid: 71cdcec7-3b69-4cb7-a8f8-1ce07fb9dc73 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:06:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-91fdc} kind: Pod name: splitter-graph-91fdc-d7f8d8f4c-q42gc namespace: kserve-ci-e2e-test resourceVersion: "26883" uid: e352b067-8f1d-4604-b3ad-4731c9c07ef5 kind: Event lastTimestamp: "2026-07-08T17:06:30Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:86bfaabcc176433317a84dc02e835b9657d32290b963c2e4ae8312aa72bd9d64" already present on machine metadata: creationTimestamp: "2026-07-08T17:06:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:06:30Z" name: splitter-graph-91fdc-d7f8d8f4c-q42gc.18c05f70e72c2736 namespace: kserve-ci-e2e-test resourceVersion: "26910" uid: 2f65cd40-28bb-4e6b-9310-a19a3819b884 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:06:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-91fdc} kind: Pod name: splitter-graph-91fdc-d7f8d8f4c-q42gc namespace: kserve-ci-e2e-test resourceVersion: "26883" uid: e352b067-8f1d-4604-b3ad-4731c9c07ef5 kind: Event lastTimestamp: "2026-07-08T17:06:31Z" message: 'Created container: splitter-graph-91fdc' metadata: creationTimestamp: "2026-07-08T17:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:06:31Z" name: splitter-graph-91fdc-d7f8d8f4c-q42gc.18c05f70eae5b3de namespace: kserve-ci-e2e-test resourceVersion: "26911" uid: 1a9b22f6-de58-4f5c-858b-563efba82c09 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:06:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-91fdc} kind: Pod name: splitter-graph-91fdc-d7f8d8f4c-q42gc namespace: kserve-ci-e2e-test resourceVersion: "26883" uid: e352b067-8f1d-4604-b3ad-4731c9c07ef5 kind: Event lastTimestamp: "2026-07-08T17:06:31Z" message: Started container splitter-graph-91fdc metadata: creationTimestamp: "2026-07-08T17:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:06:31Z" name: splitter-graph-91fdc-d7f8d8f4c-q42gc.18c05f70eb5488bf namespace: kserve-ci-e2e-test resourceVersion: "26912" uid: 589cb816-3afe-495f-9094-4509fce8d49a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:14:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-91fdc} kind: Pod name: splitter-graph-91fdc-d7f8d8f4c-q42gc namespace: kserve-ci-e2e-test resourceVersion: "26883" uid: e352b067-8f1d-4604-b3ad-4731c9c07ef5 kind: Event lastTimestamp: "2026-07-08T17:14:44Z" message: Stopping container splitter-graph-91fdc metadata: creationTimestamp: "2026-07-08T17:14:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:14:44Z" name: splitter-graph-91fdc-d7f8d8f4c-q42gc.18c05fe3d2c506d2 namespace: kserve-ci-e2e-test resourceVersion: "30340" uid: 1b6a5bb2-e784-43f5-a36b-39a8a07c72ec reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T17:14:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-91fdc} kind: Pod name: splitter-graph-91fdc-d7f8d8f4c-q42gc namespace: kserve-ci-e2e-test resourceVersion: "26883" uid: e352b067-8f1d-4604-b3ad-4731c9c07ef5 kind: Event lastTimestamp: "2026-07-08T17:15:12Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T17:14:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:15:12Z" name: splitter-graph-91fdc-d7f8d8f4c-q42gc.18c05fe482b3e267 namespace: kserve-ci-e2e-test resourceVersion: "30573" uid: 033783fa-9ca3-4e0a-a3a2-5c71dda32e83 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:06:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-91fdc-d7f8d8f4c namespace: kserve-ci-e2e-test resourceVersion: "26878" uid: ae79802a-8181-4d62-99bb-193eb7ea720e kind: Event lastTimestamp: "2026-07-08T17:06:29Z" message: 'Created pod: splitter-graph-91fdc-d7f8d8f4c-q42gc' metadata: creationTimestamp: "2026-07-08T17:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:06:29Z" name: splitter-graph-91fdc-d7f8d8f4c.18c05f70a7eab872 namespace: kserve-ci-e2e-test resourceVersion: "26884" uid: 6cd72b55-0cea-41c2-ae0b-51ee24b682a4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:06:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-91fdc namespace: kserve-ci-e2e-test resourceVersion: "26874" uid: b9923eaa-89ee-4b8d-aba6-91b8ae2de9b6 kind: Event lastTimestamp: "2026-07-08T17:06:29Z" message: Scaled up replica set splitter-graph-91fdc-d7f8d8f4c from 0 to 1 metadata: creationTimestamp: "2026-07-08T17:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:06:29Z" name: splitter-graph-91fdc.18c05f70a6c5226e namespace: kserve-ci-e2e-test resourceVersion: "26879" uid: f115e222-70b6-47da-a8ec-e336de16d222 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:06:29Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-91fdc namespace: kserve-ci-e2e-test resourceVersion: "26896" uid: 5f6b9599-4f37-4378-bd97-edf98021eb43 kind: Event lastTimestamp: "2026-07-08T17:06:29Z" message: InferenceGraph [splitter-graph-91fdc] is Ready metadata: creationTimestamp: "2026-07-08T17:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:06:29Z" name: splitter-graph-91fdc.18c05f70aca4cf60 namespace: kserve-ci-e2e-test resourceVersion: "26898" uid: 38817be6-d789-4ec2-b997-4a3ef792b277 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:06:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-91fdc namespace: kserve-ci-e2e-test resourceVersion: "26876" uid: 76572857-91c4-410d-a016-0ee69273fe6f kind: Event lastTimestamp: "2026-07-08T17:06:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:06:59Z" name: splitter-graph-91fdc.18c05f74254b8fbd namespace: kserve-ci-e2e-test resourceVersion: "27123" uid: daaa0649-867a-4a0f-be28-d0b3a3504581 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:06:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-91fdc namespace: kserve-ci-e2e-test resourceVersion: "26876" uid: 76572857-91c4-410d-a016-0ee69273fe6f kind: Event lastTimestamp: "2026-07-08T17:06:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:06:59Z" name: splitter-graph-91fdc.18c05f74259dc10a namespace: kserve-ci-e2e-test resourceVersion: "27125" uid: 624e6fa2-4405-4d86-9b93-62f8c27d5be4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:55:04.670951Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1f879-predictor-669f7c8b8b-qz6j7 namespace: kserve-ci-e2e-test resourceVersion: "20694" uid: 786486ea-9806-41d8-a8ca-8835eb3db625 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1f879-predictor-669f7c8b8b-qz6j7 to ip-10-0-132-182.ec2.internal metadata: creationTimestamp: "2026-07-08T16:55:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:55:04Z" name: success-200-isvc-1f879-predictor-669f7c8b8b-qz6j7.18c05ed11d4547b3 namespace: kserve-ci-e2e-test resourceVersion: "20703" uid: c1a665a0-fa79-4bce-93b5-970104155e76 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:05Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1f879-predictor-669f7c8b8b-qz6j7 namespace: kserve-ci-e2e-test resourceVersion: "20708" uid: 786486ea-9806-41d8-a8ca-8835eb3db625 kind: Event lastTimestamp: "2026-07-08T16:55:05Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:55:05Z" name: success-200-isvc-1f879-predictor-669f7c8b8b-qz6j7.18c05ed136fe8ef9 namespace: kserve-ci-e2e-test resourceVersion: "20748" uid: 07ebc47b-813f-46da-8736-f32f4a7dc6ae reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1f879-predictor-669f7c8b8b-qz6j7 namespace: kserve-ci-e2e-test resourceVersion: "20697" uid: 786486ea-9806-41d8-a8ca-8835eb3db625 kind: Event lastTimestamp: "2026-07-08T16:55:05Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1707" already present on machine metadata: creationTimestamp: "2026-07-08T16:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:05Z" name: success-200-isvc-1f879-predictor-669f7c8b8b-qz6j7.18c05ed13867a7f3 namespace: kserve-ci-e2e-test resourceVersion: "20750" uid: 3b4a18cb-e448-4574-8e1b-f5cb87a371b0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1f879-predictor-669f7c8b8b-qz6j7 namespace: kserve-ci-e2e-test resourceVersion: "20697" uid: 786486ea-9806-41d8-a8ca-8835eb3db625 kind: Event lastTimestamp: "2026-07-08T16:55:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:05Z" name: success-200-isvc-1f879-predictor-669f7c8b8b-qz6j7.18c05ed13b7c3040 namespace: kserve-ci-e2e-test resourceVersion: "20751" uid: 5abb02e7-cc62-4517-b0bf-b324d8ec3d8b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1f879-predictor-669f7c8b8b-qz6j7 namespace: kserve-ci-e2e-test resourceVersion: "20697" uid: 786486ea-9806-41d8-a8ca-8835eb3db625 kind: Event lastTimestamp: "2026-07-08T16:55:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:05Z" name: success-200-isvc-1f879-predictor-669f7c8b8b-qz6j7.18c05ed13be85df3 namespace: kserve-ci-e2e-test resourceVersion: "20752" uid: d1e1df33-26b2-42f7-bc6a-723826ede8b6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1f879-predictor-669f7c8b8b-qz6j7 namespace: kserve-ci-e2e-test resourceVersion: "20697" uid: 786486ea-9806-41d8-a8ca-8835eb3db625 kind: Event lastTimestamp: "2026-07-08T16:55:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:05Z" name: success-200-isvc-1f879-predictor-669f7c8b8b-qz6j7.18c05ed13bf1cb6f namespace: kserve-ci-e2e-test resourceVersion: "20753" uid: fbb1827c-04b5-408b-9311-f9c4e866ccd4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1f879-predictor-669f7c8b8b-qz6j7 namespace: kserve-ci-e2e-test resourceVersion: "20697" uid: 786486ea-9806-41d8-a8ca-8835eb3db625 kind: Event lastTimestamp: "2026-07-08T16:55:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:05Z" name: success-200-isvc-1f879-predictor-669f7c8b8b-qz6j7.18c05ed13f35e917 namespace: kserve-ci-e2e-test resourceVersion: "20758" uid: 9bfe4ee0-32aa-4f23-8c9d-463e4bb009ef reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1f879-predictor-669f7c8b8b-qz6j7 namespace: kserve-ci-e2e-test resourceVersion: "20697" uid: 786486ea-9806-41d8-a8ca-8835eb3db625 kind: Event lastTimestamp: "2026-07-08T16:55:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:05Z" name: success-200-isvc-1f879-predictor-669f7c8b8b-qz6j7.18c05ed13fa9d17a namespace: kserve-ci-e2e-test resourceVersion: "20759" uid: 5e101adb-f469-4213-837d-112352eff02d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-08T16:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1f879-predictor-669f7c8b8b-qz6j7 namespace: kserve-ci-e2e-test resourceVersion: "20697" uid: 786486ea-9806-41d8-a8ca-8835eb3db625 kind: Event lastTimestamp: "2026-07-08T17:04:21Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:21Z" name: success-200-isvc-1f879-predictor-669f7c8b8b-qz6j7.18c05ed14fb23500 namespace: kserve-ci-e2e-test resourceVersion: "25428" uid: b193a871-0f55-4365-924d-a21a171cb405 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1f879-predictor-669f7c8b8b-qz6j7 namespace: kserve-ci-e2e-test resourceVersion: "20697" uid: 786486ea-9806-41d8-a8ca-8835eb3db625 kind: Event lastTimestamp: "2026-07-08T17:04:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T17:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:19Z" name: success-200-isvc-1f879-predictor-669f7c8b8b-qz6j7.18c05f52492c1169 namespace: kserve-ci-e2e-test resourceVersion: "25327" uid: d4a74184-3de0-4107-9c1b-6323cdb44b1e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1f879-predictor-669f7c8b8b-qz6j7 namespace: kserve-ci-e2e-test resourceVersion: "20697" uid: 786486ea-9806-41d8-a8ca-8835eb3db625 kind: Event lastTimestamp: "2026-07-08T17:04:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:19Z" name: success-200-isvc-1f879-predictor-669f7c8b8b-qz6j7.18c05f52492ca9eb namespace: kserve-ci-e2e-test resourceVersion: "25330" uid: 959e131f-314c-4f71-aca7-a167232e3ed1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1f879-predictor-669f7c8b8b-qz6j7 namespace: kserve-ci-e2e-test resourceVersion: "20697" uid: 786486ea-9806-41d8-a8ca-8835eb3db625 kind: Event lastTimestamp: "2026-07-08T17:04:21Z" message: 'Readiness probe failed: Get "https://10.132.0.34:8643/healthz": dial tcp 10.132.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:04:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:21Z" name: success-200-isvc-1f879-predictor-669f7c8b8b-qz6j7.18c05f52c4266960 namespace: kserve-ci-e2e-test resourceVersion: "25427" uid: 867bd280-3a19-4321-8cf3-a066d2bfb401 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1f879-predictor-669f7c8b8b namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: f419dbe4-a9aa-4d59-9319-89088092f3ab kind: Event lastTimestamp: "2026-07-08T16:55:04Z" message: 'Created pod: success-200-isvc-1f879-predictor-669f7c8b8b-qz6j7' metadata: creationTimestamp: "2026-07-08T16:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:04Z" name: success-200-isvc-1f879-predictor-669f7c8b8b.18c05ed11c98baf2 namespace: kserve-ci-e2e-test resourceVersion: "20696" uid: b3d1c0cb-694c-44ca-8792-cd3e941dd703 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1f879-predictor namespace: kserve-ci-e2e-test resourceVersion: "20673" uid: 0df7874d-37dd-4c33-8dbe-c5c41f670e43 kind: Event lastTimestamp: "2026-07-08T16:55:04Z" message: Scaled up replica set success-200-isvc-1f879-predictor-669f7c8b8b from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:04Z" name: success-200-isvc-1f879-predictor.18c05ed11aa59549 namespace: kserve-ci-e2e-test resourceVersion: "20684" uid: 6a7ecf51-4984-4cce-a7ca-f65a71fe5dcc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:55:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1f879-predictor namespace: kserve-ci-e2e-test resourceVersion: "20676" uid: afddf4b7-9b23-4cdb-b3ae-0133b1e161d1 kind: Event lastTimestamp: "2026-07-08T16:55:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:55:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:49Z" name: success-200-isvc-1f879-predictor.18c05ed499067d30 namespace: kserve-ci-e2e-test resourceVersion: "21342" uid: 32d3554e-fe02-4a37-b2c6-33c8ff9d5429 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:55:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1f879-predictor namespace: kserve-ci-e2e-test resourceVersion: "20676" uid: afddf4b7-9b23-4cdb-b3ae-0133b1e161d1 kind: Event lastTimestamp: "2026-07-08T16:55:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:55:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:49Z" name: success-200-isvc-1f879-predictor.18c05ed499701d40 namespace: kserve-ci-e2e-test resourceVersion: "21343" uid: 36074b44-fd54-4678-9e8f-cd2463f129b2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:55:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1f879-predictor namespace: kserve-ci-e2e-test resourceVersion: "20900" uid: afddf4b7-9b23-4cdb-b3ae-0133b1e161d1 kind: Event lastTimestamp: "2026-07-08T16:56:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:55:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:56:19Z" name: success-200-isvc-1f879-predictor.18c05ed8184d46ad namespace: kserve-ci-e2e-test resourceVersion: "21656" uid: 1f474ee2-c6c9-457f-b2a6-8da54a1f3d4c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:55:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1f879-predictor namespace: kserve-ci-e2e-test resourceVersion: "20900" uid: afddf4b7-9b23-4cdb-b3ae-0133b1e161d1 kind: Event lastTimestamp: "2026-07-08T16:56:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:55:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:56:19Z" name: success-200-isvc-1f879-predictor.18c05ed818c9c76a namespace: kserve-ci-e2e-test resourceVersion: "21657" uid: 3423cb31-31fd-4c52-b78d-44ad9f4a4c1e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:55:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1f879 namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 6649aa01-a4dc-4543-a1c8-70d84b8fa905 kind: Event lastTimestamp: "2026-07-08T16:55:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:55:51Z" name: success-200-isvc-1f879.18c05ed11ab23f74 namespace: kserve-ci-e2e-test resourceVersion: "21373" uid: fc54396f-ff4e-4724-915e-8d8364a1bc7c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1f879 namespace: kserve-ci-e2e-test resourceVersion: "21371" uid: 6649aa01-a4dc-4543-a1c8-70d84b8fa905 kind: Event lastTimestamp: "2026-07-08T16:55:51Z" message: InferenceService [success-200-isvc-1f879] is Ready metadata: creationTimestamp: "2026-07-08T16:55:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:55:51Z" name: success-200-isvc-1f879.18c05edc0b315e5d namespace: kserve-ci-e2e-test resourceVersion: "21372" uid: 5876cc43-d851-4d47-998e-f0876e100463 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:54:34.192469Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3816e-predictor-5ff79f6dcd-g5g6d namespace: kserve-ci-e2e-test resourceVersion: "20239" uid: 57033467-18f8-48b4-b9b3-28a072b9b17c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3816e-predictor-5ff79f6dcd-g5g6d to ip-10-0-132-182.ec2.internal metadata: creationTimestamp: "2026-07-08T16:54:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:54:34Z" name: success-200-isvc-3816e-predictor-5ff79f6dcd-g5g6d.18c05eca049c5f51 namespace: kserve-ci-e2e-test resourceVersion: "20244" uid: bdbc6804-18b5-4f74-a914-41566665bc7f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:34Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3816e-predictor-5ff79f6dcd-g5g6d namespace: kserve-ci-e2e-test resourceVersion: "20250" uid: 57033467-18f8-48b4-b9b3-28a072b9b17c kind: Event lastTimestamp: "2026-07-08T16:54:34Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:54:34Z" name: success-200-isvc-3816e-predictor-5ff79f6dcd-g5g6d.18c05eca1e280357 namespace: kserve-ci-e2e-test resourceVersion: "20302" uid: f0ea282b-4c72-4a2c-be59-a3752a51e0ba reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3816e-predictor-5ff79f6dcd-g5g6d namespace: kserve-ci-e2e-test resourceVersion: "20242" uid: 57033467-18f8-48b4-b9b3-28a072b9b17c kind: Event lastTimestamp: "2026-07-08T16:54:34Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1707" already present on machine metadata: creationTimestamp: "2026-07-08T16:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:34Z" name: success-200-isvc-3816e-predictor-5ff79f6dcd-g5g6d.18c05eca208ffbd3 namespace: kserve-ci-e2e-test resourceVersion: "20304" uid: 25e72143-1301-4a4b-977a-8c8e3a4ff16f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3816e-predictor-5ff79f6dcd-g5g6d namespace: kserve-ci-e2e-test resourceVersion: "20242" uid: 57033467-18f8-48b4-b9b3-28a072b9b17c kind: Event lastTimestamp: "2026-07-08T16:54:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:34Z" name: success-200-isvc-3816e-predictor-5ff79f6dcd-g5g6d.18c05eca236633fa namespace: kserve-ci-e2e-test resourceVersion: "20305" uid: d51eba0e-b008-46dd-84cd-cbad91e19ec0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3816e-predictor-5ff79f6dcd-g5g6d namespace: kserve-ci-e2e-test resourceVersion: "20242" uid: 57033467-18f8-48b4-b9b3-28a072b9b17c kind: Event lastTimestamp: "2026-07-08T16:54:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:34Z" name: success-200-isvc-3816e-predictor-5ff79f6dcd-g5g6d.18c05eca23d272f6 namespace: kserve-ci-e2e-test resourceVersion: "20307" uid: e68cb321-2cde-45b7-94ca-727773f5d54d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3816e-predictor-5ff79f6dcd-g5g6d namespace: kserve-ci-e2e-test resourceVersion: "20242" uid: 57033467-18f8-48b4-b9b3-28a072b9b17c kind: Event lastTimestamp: "2026-07-08T16:54:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:34Z" name: success-200-isvc-3816e-predictor-5ff79f6dcd-g5g6d.18c05eca23db34ac namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: 50aef28d-af62-4715-8fa3-190d826628ba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3816e-predictor-5ff79f6dcd-g5g6d namespace: kserve-ci-e2e-test resourceVersion: "20242" uid: 57033467-18f8-48b4-b9b3-28a072b9b17c kind: Event lastTimestamp: "2026-07-08T16:54:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:34Z" name: success-200-isvc-3816e-predictor-5ff79f6dcd-g5g6d.18c05eca270e042a namespace: kserve-ci-e2e-test resourceVersion: "20309" uid: c68a8683-4b55-46f3-aa86-924251b30b72 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3816e-predictor-5ff79f6dcd-g5g6d namespace: kserve-ci-e2e-test resourceVersion: "20242" uid: 57033467-18f8-48b4-b9b3-28a072b9b17c kind: Event lastTimestamp: "2026-07-08T16:54:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:34Z" name: success-200-isvc-3816e-predictor-5ff79f6dcd-g5g6d.18c05eca277c3931 namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: 50b3035b-9208-4605-a8f6-f625fb2740b3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:54:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3816e-predictor-5ff79f6dcd-g5g6d namespace: kserve-ci-e2e-test resourceVersion: "20242" uid: 57033467-18f8-48b4-b9b3-28a072b9b17c kind: Event lastTimestamp: "2026-07-08T16:55:11Z" message: 'Readiness probe failed: dial tcp 10.132.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:54:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:11Z" name: success-200-isvc-3816e-predictor-5ff79f6dcd-g5g6d.18c05eca4aae9d23 namespace: kserve-ci-e2e-test resourceVersion: "20831" uid: e5f32451-8a14-4670-a5ee-6e13b407008c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3816e-predictor-5ff79f6dcd-g5g6d namespace: kserve-ci-e2e-test resourceVersion: "20242" uid: 57033467-18f8-48b4-b9b3-28a072b9b17c kind: Event lastTimestamp: "2026-07-08T16:55:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:44Z" name: success-200-isvc-3816e-predictor-5ff79f6dcd-g5g6d.18c05eda5d44bfd1 namespace: kserve-ci-e2e-test resourceVersion: "21191" uid: 9bf3ece4-1948-4aff-a03d-06e6b6406ec8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3816e-predictor-5ff79f6dcd-g5g6d namespace: kserve-ci-e2e-test resourceVersion: "20242" uid: 57033467-18f8-48b4-b9b3-28a072b9b17c kind: Event lastTimestamp: "2026-07-08T16:55:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:44Z" name: success-200-isvc-3816e-predictor-5ff79f6dcd-g5g6d.18c05eda5d45c84a namespace: kserve-ci-e2e-test resourceVersion: "21193" uid: 4a1bda6d-422a-4929-89ae-f14b9f77e477 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3816e-predictor-5ff79f6dcd-g5g6d namespace: kserve-ci-e2e-test resourceVersion: "20242" uid: 57033467-18f8-48b4-b9b3-28a072b9b17c kind: Event lastTimestamp: "2026-07-08T16:55:46Z" message: 'Readiness probe failed: Get "https://10.132.0.32:8643/healthz": dial tcp 10.132.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:55:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:46Z" name: success-200-isvc-3816e-predictor-5ff79f6dcd-g5g6d.18c05edad2cee109 namespace: kserve-ci-e2e-test resourceVersion: "21307" uid: 1b128c75-a212-4762-b814-844cfb3bf640 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3816e-predictor-5ff79f6dcd namespace: kserve-ci-e2e-test resourceVersion: "20229" uid: dc889a20-b9e8-4e56-9836-b99523f55372 kind: Event lastTimestamp: "2026-07-08T16:54:34Z" message: 'Created pod: success-200-isvc-3816e-predictor-5ff79f6dcd-g5g6d' metadata: creationTimestamp: "2026-07-08T16:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:54:34Z" name: success-200-isvc-3816e-predictor-5ff79f6dcd.18c05eca04224c3a namespace: kserve-ci-e2e-test resourceVersion: "20241" uid: e9a43e06-a158-4946-bd33-c23451c32a6a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3816e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20225" uid: 6e1095b0-d9b6-401c-ab1d-5773945c64ee kind: Event lastTimestamp: "2026-07-08T16:54:34Z" message: Scaled up replica set success-200-isvc-3816e-predictor-5ff79f6dcd from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:54:34Z" name: success-200-isvc-3816e-predictor.18c05eca01bb1954 namespace: kserve-ci-e2e-test resourceVersion: "20231" uid: 624b232c-f591-4649-a9c4-1d856bf9643f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:54:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3816e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20228" uid: 33bc24d7-052f-4e7b-bb91-29719f7fb471 kind: Event lastTimestamp: "2026-07-08T16:55:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:54:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:19Z" name: success-200-isvc-3816e-predictor.18c05ecd80bf7abe namespace: kserve-ci-e2e-test resourceVersion: "20891" uid: 3a038448-9a18-4128-b3eb-38f61a252532 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:54:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3816e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20228" uid: 33bc24d7-052f-4e7b-bb91-29719f7fb471 kind: Event lastTimestamp: "2026-07-08T16:55:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:54:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:19Z" name: success-200-isvc-3816e-predictor.18c05ecd8133e532 namespace: kserve-ci-e2e-test resourceVersion: "20892" uid: 8ceb8de0-0db9-48f9-b11f-69c19c76c7f5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:55:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3816e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20465" uid: 33bc24d7-052f-4e7b-bb91-29719f7fb471 kind: Event lastTimestamp: "2026-07-08T16:55:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:34Z" name: success-200-isvc-3816e-predictor.18c05ed10024afcb namespace: kserve-ci-e2e-test resourceVersion: "21052" uid: d5c508e7-ec86-4ba4-8cf3-b2498290a456 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:55:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3816e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20465" uid: 33bc24d7-052f-4e7b-bb91-29719f7fb471 kind: Event lastTimestamp: "2026-07-08T16:55:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:34Z" name: success-200-isvc-3816e-predictor.18c05ed100a19279 namespace: kserve-ci-e2e-test resourceVersion: "21053" uid: 76fd47d0-078b-4590-ae11-8622bdb57c00 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:54:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3816e namespace: kserve-ci-e2e-test resourceVersion: "20206" uid: 88067b30-351e-43c0-9287-ab06fa7a4d55 kind: Event lastTimestamp: "2026-07-08T16:55:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:55:21Z" name: success-200-isvc-3816e.18c05eca01e22bf5 namespace: kserve-ci-e2e-test resourceVersion: "20927" uid: 85cee1bc-2b89-4f9e-b8fc-9e6193ad8a7b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3816e namespace: kserve-ci-e2e-test resourceVersion: "20206" uid: 88067b30-351e-43c0-9287-ab06fa7a4d55 kind: Event lastTimestamp: "2026-07-08T16:54:34Z" message: 'Failed to update status for InferenceService "success-200-isvc-3816e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3816e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:54:34Z" name: success-200-isvc-3816e.18c05eca02a56d07 namespace: kserve-ci-e2e-test resourceVersion: "20236" uid: 28c654bb-a580-4c69-bc4d-460ffbc4c4f4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3816e namespace: kserve-ci-e2e-test resourceVersion: "20206" uid: 88067b30-351e-43c0-9287-ab06fa7a4d55 kind: Event lastTimestamp: "2026-07-08T16:54:34Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3816e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:54:34Z" name: success-200-isvc-3816e.18c05eca02a5d13e namespace: kserve-ci-e2e-test resourceVersion: "20238" uid: 697f7255-af4e-4aee-bfa8-a6f75d26c68f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3816e namespace: kserve-ci-e2e-test resourceVersion: "20923" uid: 88067b30-351e-43c0-9287-ab06fa7a4d55 kind: Event lastTimestamp: "2026-07-08T16:55:21Z" message: InferenceService [success-200-isvc-3816e] is Ready metadata: creationTimestamp: "2026-07-08T16:55:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:55:21Z" name: success-200-isvc-3816e.18c05ed506afddb7 namespace: kserve-ci-e2e-test resourceVersion: "20924" uid: 48d1191c-5d12-49e1-9d74-ec92fe1c3576 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T17:04:59.465428Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-73f94-predictor-68bc54bd54-h82px namespace: kserve-ci-e2e-test resourceVersion: "25794" uid: 482ae0f4-77fe-41ca-b582-1cf28a9805f6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-73f94-predictor-68bc54bd54-h82px to ip-10-0-132-182.ec2.internal metadata: creationTimestamp: "2026-07-08T17:04:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T17:04:59Z" name: success-200-isvc-73f94-predictor-68bc54bd54-h82px.18c05f5b99c88942 namespace: kserve-ci-e2e-test resourceVersion: "25800" uid: 79f5243b-cbaa-487f-9f7f-9db60d488a66 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:59Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-73f94-predictor-68bc54bd54-h82px namespace: kserve-ci-e2e-test resourceVersion: "25798" uid: 482ae0f4-77fe-41ca-b582-1cf28a9805f6 kind: Event lastTimestamp: "2026-07-08T17:04:59Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-73f94-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:59Z" name: success-200-isvc-73f94-predictor-68bc54bd54-h82px.18c05f5ba39dc72f namespace: kserve-ci-e2e-test resourceVersion: "25838" uid: 5dd1180a-fce8-4153-a41d-1247acfd1be3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:00Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-73f94-predictor-68bc54bd54-h82px namespace: kserve-ci-e2e-test resourceVersion: "25807" uid: 482ae0f4-77fe-41ca-b582-1cf28a9805f6 kind: Event lastTimestamp: "2026-07-08T17:05:00Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T17:05:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T17:05:00Z" name: success-200-isvc-73f94-predictor-68bc54bd54-h82px.18c05f5bd762b0cc namespace: kserve-ci-e2e-test resourceVersion: "25862" uid: 2afe22d5-b226-40e7-b60a-519923deddd7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-73f94-predictor-68bc54bd54-h82px namespace: kserve-ci-e2e-test resourceVersion: "25798" uid: 482ae0f4-77fe-41ca-b582-1cf28a9805f6 kind: Event lastTimestamp: "2026-07-08T17:05:00Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1707" already present on machine metadata: creationTimestamp: "2026-07-08T17:05:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:00Z" name: success-200-isvc-73f94-predictor-68bc54bd54-h82px.18c05f5bd8b92621 namespace: kserve-ci-e2e-test resourceVersion: "25865" uid: 3160178e-ebdc-421b-884b-5480991d4c11 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-73f94-predictor-68bc54bd54-h82px namespace: kserve-ci-e2e-test resourceVersion: "25798" uid: 482ae0f4-77fe-41ca-b582-1cf28a9805f6 kind: Event lastTimestamp: "2026-07-08T17:05:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T17:05:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:00Z" name: success-200-isvc-73f94-predictor-68bc54bd54-h82px.18c05f5bdc616073 namespace: kserve-ci-e2e-test resourceVersion: "25866" uid: ca22a521-0a1a-4680-99cf-3712cb68899e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-73f94-predictor-68bc54bd54-h82px namespace: kserve-ci-e2e-test resourceVersion: "25798" uid: 482ae0f4-77fe-41ca-b582-1cf28a9805f6 kind: Event lastTimestamp: "2026-07-08T17:05:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T17:05:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:00Z" name: success-200-isvc-73f94-predictor-68bc54bd54-h82px.18c05f5bdcd606cb namespace: kserve-ci-e2e-test resourceVersion: "25867" uid: 9d8e926d-51f3-4ac3-b283-5c91be3b160b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-73f94-predictor-68bc54bd54-h82px namespace: kserve-ci-e2e-test resourceVersion: "25798" uid: 482ae0f4-77fe-41ca-b582-1cf28a9805f6 kind: Event lastTimestamp: "2026-07-08T17:05:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T17:05:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:00Z" name: success-200-isvc-73f94-predictor-68bc54bd54-h82px.18c05f5bdce0a227 namespace: kserve-ci-e2e-test resourceVersion: "25868" uid: 50fdf78f-4bc4-4620-8ecb-95829d04f832 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-73f94-predictor-68bc54bd54-h82px namespace: kserve-ci-e2e-test resourceVersion: "25798" uid: 482ae0f4-77fe-41ca-b582-1cf28a9805f6 kind: Event lastTimestamp: "2026-07-08T17:05:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T17:05:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:00Z" name: success-200-isvc-73f94-predictor-68bc54bd54-h82px.18c05f5bdface775 namespace: kserve-ci-e2e-test resourceVersion: "25869" uid: 6aa9f865-1423-4bce-bda9-a1479cdcf271 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-73f94-predictor-68bc54bd54-h82px namespace: kserve-ci-e2e-test resourceVersion: "25798" uid: 482ae0f4-77fe-41ca-b582-1cf28a9805f6 kind: Event lastTimestamp: "2026-07-08T17:05:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:05:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:00Z" name: success-200-isvc-73f94-predictor-68bc54bd54-h82px.18c05f5be01e05cd namespace: kserve-ci-e2e-test resourceVersion: "25870" uid: 910d5402-7f31-408e-8f4c-3f927ef222f7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T17:05:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-73f94-predictor-68bc54bd54-h82px namespace: kserve-ci-e2e-test resourceVersion: "25798" uid: 482ae0f4-77fe-41ca-b582-1cf28a9805f6 kind: Event lastTimestamp: "2026-07-08T17:05:38Z" message: 'Readiness probe failed: dial tcp 10.132.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:05:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:38Z" name: success-200-isvc-73f94-predictor-68bc54bd54-h82px.18c05f5c365c9212 namespace: kserve-ci-e2e-test resourceVersion: "26385" uid: 0c0e9fa3-1c3f-4c3f-b805-fc7077cefc9b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:22:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-73f94-predictor-68bc54bd54-h82px namespace: kserve-ci-e2e-test resourceVersion: "25798" uid: 482ae0f4-77fe-41ca-b582-1cf28a9805f6 kind: Event lastTimestamp: "2026-07-08T17:22:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T17:22:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:22:18Z" name: success-200-isvc-73f94-predictor-68bc54bd54-h82px.18c0604d9dab6afc namespace: kserve-ci-e2e-test resourceVersion: "33495" uid: 0c20ca34-ee80-4fb1-940f-cb39deaa0d89 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:22:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-73f94-predictor-68bc54bd54-h82px namespace: kserve-ci-e2e-test resourceVersion: "25798" uid: 482ae0f4-77fe-41ca-b582-1cf28a9805f6 kind: Event lastTimestamp: "2026-07-08T17:22:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:22:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:22:18Z" name: success-200-isvc-73f94-predictor-68bc54bd54-h82px.18c0604d9dabe0fb namespace: kserve-ci-e2e-test resourceVersion: "33496" uid: ab3ff209-8e87-466e-ab4f-ee7f0a9ea19e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-73f94-predictor-68bc54bd54 namespace: kserve-ci-e2e-test resourceVersion: "25775" uid: e2effb4b-8ce2-4e98-9316-7583721e82b4 kind: Event lastTimestamp: "2026-07-08T17:04:59Z" message: 'Created pod: success-200-isvc-73f94-predictor-68bc54bd54-h82px' metadata: creationTimestamp: "2026-07-08T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:04:59Z" name: success-200-isvc-73f94-predictor-68bc54bd54.18c05f5b99596dd1 namespace: kserve-ci-e2e-test resourceVersion: "25797" uid: c1727f88-95aa-497e-a655-5dc3b65fe567 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-73f94-predictor namespace: kserve-ci-e2e-test resourceVersion: "25772" uid: a0882f1f-d615-432d-acdc-e2e1f237404b kind: Event lastTimestamp: "2026-07-08T17:04:59Z" message: Scaled up replica set success-200-isvc-73f94-predictor-68bc54bd54 from 0 to 1 metadata: creationTimestamp: "2026-07-08T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:04:59Z" name: success-200-isvc-73f94-predictor.18c05f5b96e78b11 namespace: kserve-ci-e2e-test resourceVersion: "25782" uid: 3d412f52-23fd-44d2-8c4a-62a52d243e21 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-73f94-predictor namespace: kserve-ci-e2e-test resourceVersion: "25774" uid: 88840f37-4e6d-4949-88dc-b2431e7a098c kind: Event lastTimestamp: "2026-07-08T17:05:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:05:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:05:14Z" name: success-200-isvc-73f94-predictor.18c05f5f160d4254 namespace: kserve-ci-e2e-test resourceVersion: "26008" uid: d9af3449-f64a-4b2d-a637-3ae7b3d18b90 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-73f94-predictor namespace: kserve-ci-e2e-test resourceVersion: "25774" uid: 88840f37-4e6d-4949-88dc-b2431e7a098c kind: Event lastTimestamp: "2026-07-08T17:05:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:05:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:05:14Z" name: success-200-isvc-73f94-predictor.18c05f5f167aa2e7 namespace: kserve-ci-e2e-test resourceVersion: "26009" uid: e301bcfd-91c8-47b7-bfca-c4bf739b9865 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T17:05:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-73f94-predictor namespace: kserve-ci-e2e-test resourceVersion: "26007" uid: 88840f37-4e6d-4949-88dc-b2431e7a098c kind: Event lastTimestamp: "2026-07-08T17:06:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:06:14Z" name: success-200-isvc-73f94-predictor.18c05f629540d02a namespace: kserve-ci-e2e-test resourceVersion: "26730" uid: f4603605-813e-4ae9-aeac-a2256f50cfe1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T17:05:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-73f94-predictor namespace: kserve-ci-e2e-test resourceVersion: "26007" uid: 88840f37-4e6d-4949-88dc-b2431e7a098c kind: Event lastTimestamp: "2026-07-08T17:06:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:06:14Z" name: success-200-isvc-73f94-predictor.18c05f6295d0d7f1 namespace: kserve-ci-e2e-test resourceVersion: "26731" uid: 43a7dcd7-c620-4b25-acab-b6a245d0f601 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T17:04:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-73f94 namespace: kserve-ci-e2e-test resourceVersion: "25750" uid: 80df5c24-6542-44d1-a921-6accdd182292 kind: Event lastTimestamp: "2026-07-08T17:05:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:05:48Z" name: success-200-isvc-73f94.18c05f5b96f5cf49 namespace: kserve-ci-e2e-test resourceVersion: "26485" uid: 77db98a9-b624-4733-94b1-b3221a4b2bf7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-73f94 namespace: kserve-ci-e2e-test resourceVersion: "26483" uid: 80df5c24-6542-44d1-a921-6accdd182292 kind: Event lastTimestamp: "2026-07-08T17:05:48Z" message: InferenceService [success-200-isvc-73f94] is Ready metadata: creationTimestamp: "2026-07-08T17:05:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:05:48Z" name: success-200-isvc-73f94.18c05f66f1df9a19 namespace: kserve-ci-e2e-test resourceVersion: "26484" uid: c033d36e-47e3-4306-88dd-fa28ecdb5b63 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T17:04:19.532304Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6 namespace: kserve-ci-e2e-test resourceVersion: "25338" uid: 1389556b-cd1b-4312-8ed0-798eb53093b4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6 to ip-10-0-132-182.ec2.internal metadata: creationTimestamp: "2026-07-08T17:04:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T17:04:19Z" name: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6.18c05f524d95678a namespace: kserve-ci-e2e-test resourceVersion: "25352" uid: 5834bfcd-4013-4bc1-97ed-b780eb1f0165 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:04:19Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6 namespace: kserve-ci-e2e-test resourceVersion: "25341" uid: 1389556b-cd1b-4312-8ed0-798eb53093b4 kind: Event lastTimestamp: "2026-07-08T17:04:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-78c4a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T17:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:20Z" name: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6.18c05f5257e48c47 namespace: kserve-ci-e2e-test resourceVersion: "25407" uid: b4ee2633-9675-41bf-bfc2-75d112332297 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:21Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6 namespace: kserve-ci-e2e-test resourceVersion: "25366" uid: 1389556b-cd1b-4312-8ed0-798eb53093b4 kind: Event lastTimestamp: "2026-07-08T17:04:21Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T17:04:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T17:04:21Z" name: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6.18c05f52c1df499f namespace: kserve-ci-e2e-test resourceVersion: "25424" uid: 6c03dd24-7635-43b2-b23a-bffe8b679049 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6 namespace: kserve-ci-e2e-test resourceVersion: "25341" uid: 1389556b-cd1b-4312-8ed0-798eb53093b4 kind: Event lastTimestamp: "2026-07-08T17:04:21Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1707" already present on machine metadata: creationTimestamp: "2026-07-08T17:04:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:21Z" name: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6.18c05f52c32a1030 namespace: kserve-ci-e2e-test resourceVersion: "25426" uid: 41cdec6b-878e-4dac-be0c-bde4fc1ce5ea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6 namespace: kserve-ci-e2e-test resourceVersion: "25341" uid: 1389556b-cd1b-4312-8ed0-798eb53093b4 kind: Event lastTimestamp: "2026-07-08T17:04:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T17:04:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:21Z" name: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6.18c05f52c65ed4c6 namespace: kserve-ci-e2e-test resourceVersion: "25429" uid: 5e877f21-f22a-4ced-a24e-a05b86e8f5cc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6 namespace: kserve-ci-e2e-test resourceVersion: "25341" uid: 1389556b-cd1b-4312-8ed0-798eb53093b4 kind: Event lastTimestamp: "2026-07-08T17:04:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T17:04:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:21Z" name: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6.18c05f52c6d14b4b namespace: kserve-ci-e2e-test resourceVersion: "25430" uid: 015131e7-9fb6-4d69-81b6-e6b7bac333b7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6 namespace: kserve-ci-e2e-test resourceVersion: "25341" uid: 1389556b-cd1b-4312-8ed0-798eb53093b4 kind: Event lastTimestamp: "2026-07-08T17:04:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T17:04:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:21Z" name: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6.18c05f52c6dc7538 namespace: kserve-ci-e2e-test resourceVersion: "25431" uid: 9ac0d6b4-fea7-4403-b3f5-ce5401ed9ef4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6 namespace: kserve-ci-e2e-test resourceVersion: "25341" uid: 1389556b-cd1b-4312-8ed0-798eb53093b4 kind: Event lastTimestamp: "2026-07-08T17:04:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T17:04:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:21Z" name: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6.18c05f52cb3c7ec5 namespace: kserve-ci-e2e-test resourceVersion: "25433" uid: 79df20b2-c6a6-4f84-9504-eced19848c6e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6 namespace: kserve-ci-e2e-test resourceVersion: "25341" uid: 1389556b-cd1b-4312-8ed0-798eb53093b4 kind: Event lastTimestamp: "2026-07-08T17:04:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:04:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:21Z" name: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6.18c05f52cbb4bbc0 namespace: kserve-ci-e2e-test resourceVersion: "25434" uid: 9b6aac7c-1e83-4038-8635-c514447c21a4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T17:04:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6 namespace: kserve-ci-e2e-test resourceVersion: "25341" uid: 1389556b-cd1b-4312-8ed0-798eb53093b4 kind: Event lastTimestamp: "2026-07-08T17:04:57Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:04:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:57Z" name: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6.18c05f52db4c7a4f namespace: kserve-ci-e2e-test resourceVersion: "25719" uid: 46c8e4b3-f8c7-4de4-8af2-06012087a9a9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6 namespace: kserve-ci-e2e-test resourceVersion: "25341" uid: 1389556b-cd1b-4312-8ed0-798eb53093b4 kind: Event lastTimestamp: "2026-07-08T17:05:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T17:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:29Z" name: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6.18c05f62a723c9d1 namespace: kserve-ci-e2e-test resourceVersion: "26207" uid: e6645ba2-c9ae-43d0-9ea0-1bc64a4b2dd6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6 namespace: kserve-ci-e2e-test resourceVersion: "25341" uid: 1389556b-cd1b-4312-8ed0-798eb53093b4 kind: Event lastTimestamp: "2026-07-08T17:05:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:29Z" name: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6.18c05f62a72456ff namespace: kserve-ci-e2e-test resourceVersion: "26209" uid: a5fa41db-8c69-49a9-af88-3e47e39d8fc3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6 namespace: kserve-ci-e2e-test resourceVersion: "25341" uid: 1389556b-cd1b-4312-8ed0-798eb53093b4 kind: Event lastTimestamp: "2026-07-08T17:05:33Z" message: 'Readiness probe failed: Get "https://10.132.0.39:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-07-08T17:05:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:33Z" name: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6.18c05f639f24b9dc namespace: kserve-ci-e2e-test resourceVersion: "26354" uid: a18068c2-5437-414d-8085-75591cd0f8cd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-78c4a-predictor-7cc8c8bf4b namespace: kserve-ci-e2e-test resourceVersion: "25325" uid: 0cc8b2f4-d4ca-48a4-8bda-e659d6d651fb kind: Event lastTimestamp: "2026-07-08T17:04:19Z" message: 'Created pod: success-200-isvc-78c4a-predictor-7cc8c8bf4b-mxbm6' metadata: creationTimestamp: "2026-07-08T17:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:04:19Z" name: success-200-isvc-78c4a-predictor-7cc8c8bf4b.18c05f524cff1fd0 namespace: kserve-ci-e2e-test resourceVersion: "25340" uid: 8cb92a88-f93d-4f9b-8050-2fa3104269ce reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-78c4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25320" uid: d937766f-c5aa-4770-bfd5-580bc510a55d kind: Event lastTimestamp: "2026-07-08T17:04:19Z" message: Scaled up replica set success-200-isvc-78c4a-predictor-7cc8c8bf4b from 0 to 1 metadata: creationTimestamp: "2026-07-08T17:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:04:19Z" name: success-200-isvc-78c4a-predictor.18c05f52491db7d3 namespace: kserve-ci-e2e-test resourceVersion: "25326" uid: efc2affd-83cc-44a7-b912-c4fdbcdfa269 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-78c4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25323" uid: e4558bd6-b4a4-4961-bda0-4060309f2f3f kind: Event lastTimestamp: "2026-07-08T17:04:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:04:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:04:34Z" name: success-200-isvc-78c4a-predictor.18c05f55c7d7b283 namespace: kserve-ci-e2e-test resourceVersion: "25542" uid: 4030e13d-668f-43f4-ac8c-9f0837aa30a7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-78c4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25323" uid: e4558bd6-b4a4-4961-bda0-4060309f2f3f kind: Event lastTimestamp: "2026-07-08T17:04:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:04:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:04:34Z" name: success-200-isvc-78c4a-predictor.18c05f55c8412173 namespace: kserve-ci-e2e-test resourceVersion: "25544" uid: 08d48f04-fcf5-44ef-be23-6d662b0fbb14 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T17:04:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-78c4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25543" uid: e4558bd6-b4a4-4961-bda0-4060309f2f3f kind: Event lastTimestamp: "2026-07-08T17:05:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:04:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:05:19Z" name: success-200-isvc-78c4a-predictor.18c05f5946fe0341 namespace: kserve-ci-e2e-test resourceVersion: "26055" uid: fa4e36b2-7ad1-4c88-a183-78bd11558e11 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T17:04:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-78c4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25543" uid: e4558bd6-b4a4-4961-bda0-4060309f2f3f kind: Event lastTimestamp: "2026-07-08T17:05:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:04:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:05:19Z" name: success-200-isvc-78c4a-predictor.18c05f59473fd578 namespace: kserve-ci-e2e-test resourceVersion: "26057" uid: 33461c04-52f0-4163-8a91-3c82e611c778 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T17:04:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-78c4a namespace: kserve-ci-e2e-test resourceVersion: "25303" uid: 82385ff0-3318-4ddb-9826-26a0b7ac10d4 kind: Event lastTimestamp: "2026-07-08T17:05:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T17:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:05:08Z" name: success-200-isvc-78c4a.18c05f52495695d9 namespace: kserve-ci-e2e-test resourceVersion: "25959" uid: 45cd9781-3f15-4937-a478-eeceb72d8f05 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-78c4a namespace: kserve-ci-e2e-test resourceVersion: "25303" uid: 82385ff0-3318-4ddb-9826-26a0b7ac10d4 kind: Event lastTimestamp: "2026-07-08T17:04:19Z" message: 'Failed to update status for InferenceService "success-200-isvc-78c4a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-78c4a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T17:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:04:19Z" name: success-200-isvc-78c4a.18c05f5249bb9429 namespace: kserve-ci-e2e-test resourceVersion: "25331" uid: 2afeabbe-5ea4-43e2-aa47-f2c119f4aa5d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-78c4a namespace: kserve-ci-e2e-test resourceVersion: "25303" uid: 82385ff0-3318-4ddb-9826-26a0b7ac10d4 kind: Event lastTimestamp: "2026-07-08T17:04:19Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-78c4a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T17:04:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:04:19Z" name: success-200-isvc-78c4a.18c05f5249bc79c0 namespace: kserve-ci-e2e-test resourceVersion: "25335" uid: 85de0829-8356-47d9-9511-bdeed79a5269 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-78c4a namespace: kserve-ci-e2e-test resourceVersion: "25957" uid: 82385ff0-3318-4ddb-9826-26a0b7ac10d4 kind: Event lastTimestamp: "2026-07-08T17:05:08Z" message: InferenceService [success-200-isvc-78c4a] is Ready metadata: creationTimestamp: "2026-07-08T17:05:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:05:08Z" name: success-200-isvc-78c4a.18c05f5d97a06f03 namespace: kserve-ci-e2e-test resourceVersion: "25958" uid: 1090e482-98b3-49c6-95c2-2c42529f9558 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:53:54.432960Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8befa-predictor-85866c45b8-tgk8s namespace: kserve-ci-e2e-test resourceVersion: "19797" uid: c0601e45-bb2c-453e-9922-303007fe8fba kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8befa-predictor-85866c45b8-tgk8s to ip-10-0-132-182.ec2.internal metadata: creationTimestamp: "2026-07-08T16:53:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:53:54Z" name: success-200-isvc-8befa-predictor-85866c45b8-tgk8s.18c05ec0c2c2653a namespace: kserve-ci-e2e-test resourceVersion: "19805" uid: 00c0d816-44ff-415f-9a84-8099f40c82d1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:54Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8befa-predictor-85866c45b8-tgk8s namespace: kserve-ci-e2e-test resourceVersion: "19811" uid: c0601e45-bb2c-453e-9922-303007fe8fba kind: Event lastTimestamp: "2026-07-08T16:53:54Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:53:54Z" name: success-200-isvc-8befa-predictor-85866c45b8-tgk8s.18c05ec0dcaee881 namespace: kserve-ci-e2e-test resourceVersion: "19848" uid: 4045e42a-8d1c-4499-921f-cc598a21b901 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8befa-predictor-85866c45b8-tgk8s namespace: kserve-ci-e2e-test resourceVersion: "19801" uid: c0601e45-bb2c-453e-9922-303007fe8fba kind: Event lastTimestamp: "2026-07-08T16:53:54Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1707" already present on machine metadata: creationTimestamp: "2026-07-08T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:54Z" name: success-200-isvc-8befa-predictor-85866c45b8-tgk8s.18c05ec0de63d0f2 namespace: kserve-ci-e2e-test resourceVersion: "19851" uid: ef7f2f46-8d7c-40a4-b9b5-f516caecf526 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8befa-predictor-85866c45b8-tgk8s namespace: kserve-ci-e2e-test resourceVersion: "19801" uid: c0601e45-bb2c-453e-9922-303007fe8fba kind: Event lastTimestamp: "2026-07-08T16:53:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:54Z" name: success-200-isvc-8befa-predictor-85866c45b8-tgk8s.18c05ec0e12f321a namespace: kserve-ci-e2e-test resourceVersion: "19853" uid: 24418867-71d9-4f7e-b0e0-9d829c2c0cba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8befa-predictor-85866c45b8-tgk8s namespace: kserve-ci-e2e-test resourceVersion: "19801" uid: c0601e45-bb2c-453e-9922-303007fe8fba kind: Event lastTimestamp: "2026-07-08T16:53:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:54Z" name: success-200-isvc-8befa-predictor-85866c45b8-tgk8s.18c05ec0e1bfd6c8 namespace: kserve-ci-e2e-test resourceVersion: "19854" uid: 1fcaf41a-768c-4ec5-a734-181097c6ef3d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8befa-predictor-85866c45b8-tgk8s namespace: kserve-ci-e2e-test resourceVersion: "19801" uid: c0601e45-bb2c-453e-9922-303007fe8fba kind: Event lastTimestamp: "2026-07-08T16:53:54Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:54Z" name: success-200-isvc-8befa-predictor-85866c45b8-tgk8s.18c05ec0e1c994fa namespace: kserve-ci-e2e-test resourceVersion: "19855" uid: f6bbd169-e577-426e-9107-420affdc6043 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8befa-predictor-85866c45b8-tgk8s namespace: kserve-ci-e2e-test resourceVersion: "19801" uid: c0601e45-bb2c-453e-9922-303007fe8fba kind: Event lastTimestamp: "2026-07-08T16:53:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:55Z" name: success-200-isvc-8befa-predictor-85866c45b8-tgk8s.18c05ec0e47c583f namespace: kserve-ci-e2e-test resourceVersion: "19857" uid: 1bccb096-363b-461b-85d0-54c043ba6f56 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8befa-predictor-85866c45b8-tgk8s namespace: kserve-ci-e2e-test resourceVersion: "19801" uid: c0601e45-bb2c-453e-9922-303007fe8fba kind: Event lastTimestamp: "2026-07-08T16:53:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:55Z" name: success-200-isvc-8befa-predictor-85866c45b8-tgk8s.18c05ec0e4ee0a84 namespace: kserve-ci-e2e-test resourceVersion: "19858" uid: 6ee9114e-7637-4aba-bfd5-e01b7f6efab6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:53:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8befa-predictor-85866c45b8-tgk8s namespace: kserve-ci-e2e-test resourceVersion: "19801" uid: c0601e45-bb2c-453e-9922-303007fe8fba kind: Event lastTimestamp: "2026-07-08T16:54:31Z" message: 'Readiness probe failed: dial tcp 10.132.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:31Z" name: success-200-isvc-8befa-predictor-85866c45b8-tgk8s.18c05ec0efc12299 namespace: kserve-ci-e2e-test resourceVersion: "20161" uid: da774a33-a038-44c7-b17a-b701934939a2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8befa-predictor-85866c45b8-tgk8s namespace: kserve-ci-e2e-test resourceVersion: "19801" uid: c0601e45-bb2c-453e-9922-303007fe8fba kind: Event lastTimestamp: "2026-07-08T16:55:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:04Z" name: success-200-isvc-8befa-predictor-85866c45b8-tgk8s.18c05ed1192f4def namespace: kserve-ci-e2e-test resourceVersion: "20670" uid: 98b7eea9-c27f-40ea-aae2-ee413f78c65f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8befa-predictor-85866c45b8-tgk8s namespace: kserve-ci-e2e-test resourceVersion: "19801" uid: c0601e45-bb2c-453e-9922-303007fe8fba kind: Event lastTimestamp: "2026-07-08T16:55:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:04Z" name: success-200-isvc-8befa-predictor-85866c45b8-tgk8s.18c05ed1192fccdb namespace: kserve-ci-e2e-test resourceVersion: "20675" uid: 7b22a7ee-83cd-4e3b-a9ee-8a430b09a5b4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8befa-predictor-85866c45b8-tgk8s namespace: kserve-ci-e2e-test resourceVersion: "19801" uid: c0601e45-bb2c-453e-9922-303007fe8fba kind: Event lastTimestamp: "2026-07-08T16:55:06Z" message: 'Readiness probe failed: Get "https://10.132.0.31:8643/healthz": dial tcp 10.132.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:55:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:06Z" name: success-200-isvc-8befa-predictor-85866c45b8-tgk8s.18c05ed177e56c41 namespace: kserve-ci-e2e-test resourceVersion: "20784" uid: 43054c7e-d80d-4295-88f6-637a404bb24b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8befa-predictor-85866c45b8 namespace: kserve-ci-e2e-test resourceVersion: "19781" uid: aaf6d1d0-a161-48c2-b26b-cfdaa411cd3b kind: Event lastTimestamp: "2026-07-08T16:53:54Z" message: 'Created pod: success-200-isvc-8befa-predictor-85866c45b8-tgk8s' metadata: creationTimestamp: "2026-07-08T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:53:54Z" name: success-200-isvc-8befa-predictor-85866c45b8.18c05ec0c2269d1b namespace: kserve-ci-e2e-test resourceVersion: "19799" uid: e2240ada-c3b1-418b-a447-f67bd697a7e2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8befa-predictor namespace: kserve-ci-e2e-test resourceVersion: "19779" uid: 46e0bc72-d878-4b1b-88e2-bb6ee5c61636 kind: Event lastTimestamp: "2026-07-08T16:53:54Z" message: Scaled up replica set success-200-isvc-8befa-predictor-85866c45b8 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:53:54Z" name: success-200-isvc-8befa-predictor.18c05ec0bdbcccc7 namespace: kserve-ci-e2e-test resourceVersion: "19782" uid: 867e3d95-9db8-4735-ba87-ac99084ea729 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:54:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8befa-predictor namespace: kserve-ci-e2e-test resourceVersion: "19780" uid: 6371ecfe-cc9e-43b9-ad29-4f7d6fd8cbbd kind: Event lastTimestamp: "2026-07-08T16:54:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:54:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:54:24Z" name: success-200-isvc-8befa-predictor.18c05ec43c5339cf namespace: kserve-ci-e2e-test resourceVersion: "20106" uid: eecb0ec2-2469-46de-ad3e-59763f34c72d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:54:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8befa-predictor namespace: kserve-ci-e2e-test resourceVersion: "19780" uid: 6371ecfe-cc9e-43b9-ad29-4f7d6fd8cbbd kind: Event lastTimestamp: "2026-07-08T16:54:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:54:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:54:24Z" name: success-200-isvc-8befa-predictor.18c05ec43cb8f63f namespace: kserve-ci-e2e-test resourceVersion: "20107" uid: 7729d675-9227-4bb5-ab13-170d73b6199d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:54:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8befa-predictor namespace: kserve-ci-e2e-test resourceVersion: "19999" uid: 6371ecfe-cc9e-43b9-ad29-4f7d6fd8cbbd kind: Event lastTimestamp: "2026-07-08T16:54:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:54:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:54:54Z" name: success-200-isvc-8befa-predictor.18c05ecb3b478132 namespace: kserve-ci-e2e-test resourceVersion: "20531" uid: 93f6e4d3-6aa9-4fbe-95ee-3ee7d8cbe2d9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:54:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8befa-predictor namespace: kserve-ci-e2e-test resourceVersion: "19999" uid: 6371ecfe-cc9e-43b9-ad29-4f7d6fd8cbbd kind: Event lastTimestamp: "2026-07-08T16:54:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:54:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:54:54Z" name: success-200-isvc-8befa-predictor.18c05ecb3bc93481 namespace: kserve-ci-e2e-test resourceVersion: "20533" uid: 13d34ff8-014e-4ea7-a720-31e3ee863f29 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:53:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8befa namespace: kserve-ci-e2e-test resourceVersion: "19767" uid: 6cf057ce-ffa7-4873-9039-92a87a9819a8 kind: Event lastTimestamp: "2026-07-08T16:54:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:54:41Z" name: success-200-isvc-8befa.18c05ec0bdcee700 namespace: kserve-ci-e2e-test resourceVersion: "20398" uid: 50df913b-0c68-4ca7-9399-3760b2e3d4b0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8befa namespace: kserve-ci-e2e-test resourceVersion: "20395" uid: 6cf057ce-ffa7-4873-9039-92a87a9819a8 kind: Event lastTimestamp: "2026-07-08T16:54:41Z" message: InferenceService [success-200-isvc-8befa] is Ready metadata: creationTimestamp: "2026-07-08T16:54:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:54:41Z" name: success-200-isvc-8befa.18c05ecbab6a5dff namespace: kserve-ci-e2e-test resourceVersion: "20396" uid: 261d1a7d-a4f1-4893-98fa-e43096c8d9dc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:44:39.425519Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8ceba-predictor-5567b9d7d6-cqvc4 namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: d5794e7e-1a94-4ccd-b00d-8d9adbea100a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8ceba-predictor-5567b9d7d6-cqvc4 to ip-10-0-132-182.ec2.internal metadata: creationTimestamp: "2026-07-08T16:44:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:44:39Z" name: success-200-isvc-8ceba-predictor-5567b9d7d6-cqvc4.18c05e3f89bcf34b namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: 59a58601-d2aa-4dac-8d34-1431170708eb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:39Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8ceba-predictor-5567b9d7d6-cqvc4 namespace: kserve-ci-e2e-test resourceVersion: "15419" uid: d5794e7e-1a94-4ccd-b00d-8d9adbea100a kind: Event lastTimestamp: "2026-07-08T16:44:39Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:44:39Z" name: success-200-isvc-8ceba-predictor-5567b9d7d6-cqvc4.18c05e3fa339776d namespace: kserve-ci-e2e-test resourceVersion: "15459" uid: 4e1faaf9-a0ab-40b5-9277-4a9f72a04cd5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8ceba-predictor-5567b9d7d6-cqvc4 namespace: kserve-ci-e2e-test resourceVersion: "15399" uid: d5794e7e-1a94-4ccd-b00d-8d9adbea100a kind: Event lastTimestamp: "2026-07-08T16:44:39Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1707" already present on machine metadata: creationTimestamp: "2026-07-08T16:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:39Z" name: success-200-isvc-8ceba-predictor-5567b9d7d6-cqvc4.18c05e3fa4b71ba5 namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: 8efc82bd-db1e-4165-8bd2-2457a830ee06 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8ceba-predictor-5567b9d7d6-cqvc4 namespace: kserve-ci-e2e-test resourceVersion: "15399" uid: d5794e7e-1a94-4ccd-b00d-8d9adbea100a kind: Event lastTimestamp: "2026-07-08T16:44:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:39Z" name: success-200-isvc-8ceba-predictor-5567b9d7d6-cqvc4.18c05e3fa73d7cac namespace: kserve-ci-e2e-test resourceVersion: "15463" uid: 5f362f09-1ffa-44e8-836b-cbf397cc6f57 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8ceba-predictor-5567b9d7d6-cqvc4 namespace: kserve-ci-e2e-test resourceVersion: "15399" uid: d5794e7e-1a94-4ccd-b00d-8d9adbea100a kind: Event lastTimestamp: "2026-07-08T16:44:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:39Z" name: success-200-isvc-8ceba-predictor-5567b9d7d6-cqvc4.18c05e3fa798894c namespace: kserve-ci-e2e-test resourceVersion: "15464" uid: 8b4a8026-e7eb-4d53-8c04-db10ca6c3d66 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8ceba-predictor-5567b9d7d6-cqvc4 namespace: kserve-ci-e2e-test resourceVersion: "15399" uid: d5794e7e-1a94-4ccd-b00d-8d9adbea100a kind: Event lastTimestamp: "2026-07-08T16:44:39Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:39Z" name: success-200-isvc-8ceba-predictor-5567b9d7d6-cqvc4.18c05e3fa7a1ab4f namespace: kserve-ci-e2e-test resourceVersion: "15465" uid: 1358a3cf-70a5-493d-8846-649b08bf6f99 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8ceba-predictor-5567b9d7d6-cqvc4 namespace: kserve-ci-e2e-test resourceVersion: "15399" uid: d5794e7e-1a94-4ccd-b00d-8d9adbea100a kind: Event lastTimestamp: "2026-07-08T16:44:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:39Z" name: success-200-isvc-8ceba-predictor-5567b9d7d6-cqvc4.18c05e3faa34e777 namespace: kserve-ci-e2e-test resourceVersion: "15469" uid: 5af96680-9cb4-4bd6-a11b-858cd9f676dd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8ceba-predictor-5567b9d7d6-cqvc4 namespace: kserve-ci-e2e-test resourceVersion: "15399" uid: d5794e7e-1a94-4ccd-b00d-8d9adbea100a kind: Event lastTimestamp: "2026-07-08T16:44:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:39Z" name: success-200-isvc-8ceba-predictor-5567b9d7d6-cqvc4.18c05e3faa969314 namespace: kserve-ci-e2e-test resourceVersion: "15470" uid: 030ecac7-ffcb-473a-ae5d-296e8ee99c62 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-08T16:44:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8ceba-predictor-5567b9d7d6-cqvc4 namespace: kserve-ci-e2e-test resourceVersion: "15399" uid: d5794e7e-1a94-4ccd-b00d-8d9adbea100a kind: Event lastTimestamp: "2026-07-08T16:53:56Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:44:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:56Z" name: success-200-isvc-8ceba-predictor-5567b9d7d6-cqvc4.18c05e3fdeb0d44a namespace: kserve-ci-e2e-test resourceVersion: "19890" uid: 179899b3-4ca8-474e-aeec-b7966c09ff24 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8ceba-predictor-5567b9d7d6-cqvc4 namespace: kserve-ci-e2e-test resourceVersion: "15399" uid: d5794e7e-1a94-4ccd-b00d-8d9adbea100a kind: Event lastTimestamp: "2026-07-08T16:53:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:54Z" name: success-200-isvc-8ceba-predictor-5567b9d7d6-cqvc4.18c05ec0ba591a76 namespace: kserve-ci-e2e-test resourceVersion: "19771" uid: 5af5589f-c350-49ec-830f-e46b3954df6c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8ceba-predictor-5567b9d7d6-cqvc4 namespace: kserve-ci-e2e-test resourceVersion: "15399" uid: d5794e7e-1a94-4ccd-b00d-8d9adbea100a kind: Event lastTimestamp: "2026-07-08T16:53:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:54Z" name: success-200-isvc-8ceba-predictor-5567b9d7d6-cqvc4.18c05ec0ba5976e6 namespace: kserve-ci-e2e-test resourceVersion: "19772" uid: 128b85ce-77d4-40a7-ad36-307bb1a74c99 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8ceba-predictor-5567b9d7d6-cqvc4 namespace: kserve-ci-e2e-test resourceVersion: "15399" uid: d5794e7e-1a94-4ccd-b00d-8d9adbea100a kind: Event lastTimestamp: "2026-07-08T16:53:56Z" message: 'Readiness probe failed: Get "https://10.132.0.26:8643/healthz": dial tcp 10.132.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:56Z" name: success-200-isvc-8ceba-predictor-5567b9d7d6-cqvc4.18c05ec1531c03b2 namespace: kserve-ci-e2e-test resourceVersion: "19889" uid: ccd15121-7a77-4a3c-ad09-951bb662ea8b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8ceba-predictor-5567b9d7d6 namespace: kserve-ci-e2e-test resourceVersion: "15384" uid: a249c7d4-9fde-4640-b3dd-11b1ab8ef70b kind: Event lastTimestamp: "2026-07-08T16:44:39Z" message: 'Created pod: success-200-isvc-8ceba-predictor-5567b9d7d6-cqvc4' metadata: creationTimestamp: "2026-07-08T16:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:44:39Z" name: success-200-isvc-8ceba-predictor-5567b9d7d6.18c05e3f892f7551 namespace: kserve-ci-e2e-test resourceVersion: "15398" uid: 196c5d7d-359a-42f5-bdc2-9167f41a791a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8ceba-predictor namespace: kserve-ci-e2e-test resourceVersion: "15378" uid: c79da7f9-1ed7-4560-9b14-0391120df035 kind: Event lastTimestamp: "2026-07-08T16:44:39Z" message: Scaled up replica set success-200-isvc-8ceba-predictor-5567b9d7d6 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:44:39Z" name: success-200-isvc-8ceba-predictor.18c05e3f85e53169 namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 84d13d72-ece1-4604-8a33-6f67efeca2af reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:44:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8ceba-predictor namespace: kserve-ci-e2e-test resourceVersion: "15383" uid: c0e7cf02-c67d-4bc4-ad3d-00813f6784d6 kind: Event lastTimestamp: "2026-07-08T16:45:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:44:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:45:09Z" name: success-200-isvc-8ceba-predictor.18c05e430465fd02 namespace: kserve-ci-e2e-test resourceVersion: "15780" uid: d2d66a82-2e85-415f-b1bb-6bd535ba9275 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:44:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8ceba-predictor namespace: kserve-ci-e2e-test resourceVersion: "15383" uid: c0e7cf02-c67d-4bc4-ad3d-00813f6784d6 kind: Event lastTimestamp: "2026-07-08T16:45:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:44:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:45:09Z" name: success-200-isvc-8ceba-predictor.18c05e4304da267e namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: 38a5b1f1-598d-4481-8400-7035d0b97a06 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:45:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8ceba-predictor namespace: kserve-ci-e2e-test resourceVersion: "15629" uid: c0e7cf02-c67d-4bc4-ad3d-00813f6784d6 kind: Event lastTimestamp: "2026-07-08T16:45:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:45:39Z" name: success-200-isvc-8ceba-predictor.18c05e4a02913724 namespace: kserve-ci-e2e-test resourceVersion: "16220" uid: 42f4c129-0fff-492b-986c-18a2b73c6f2b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:45:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8ceba-predictor namespace: kserve-ci-e2e-test resourceVersion: "15629" uid: c0e7cf02-c67d-4bc4-ad3d-00813f6784d6 kind: Event lastTimestamp: "2026-07-08T16:45:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:45:39Z" name: success-200-isvc-8ceba-predictor.18c05e4a0306d872 namespace: kserve-ci-e2e-test resourceVersion: "16222" uid: f00a45b0-960c-469b-8141-726493f08d80 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:44:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8ceba namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 98796351-3899-4b43-9f15-3fa3f669e1e8 kind: Event lastTimestamp: "2026-07-08T16:45:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:45:27Z" name: success-200-isvc-8ceba.18c05e3f85f16ba3 namespace: kserve-ci-e2e-test resourceVersion: "16117" uid: c2738dbe-9c24-4dd1-b904-ebb69b07c596 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8ceba namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 98796351-3899-4b43-9f15-3fa3f669e1e8 kind: Event lastTimestamp: "2026-07-08T16:44:39Z" message: 'Failed to update status for InferenceService "success-200-isvc-8ceba": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8ceba": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:44:39Z" name: success-200-isvc-8ceba.18c05e3f8671a106 namespace: kserve-ci-e2e-test resourceVersion: "15391" uid: ec7b0851-9ccf-4bf3-b2f9-dc8e39aeb94e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8ceba namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 98796351-3899-4b43-9f15-3fa3f669e1e8 kind: Event lastTimestamp: "2026-07-08T16:44:39Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8ceba": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:44:39Z" name: success-200-isvc-8ceba.18c05e3f8671f6ff namespace: kserve-ci-e2e-test resourceVersion: "15393" uid: 7f6aa5a4-7fb3-4791-acb4-8de526da553f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8ceba namespace: kserve-ci-e2e-test resourceVersion: "16114" uid: 98796351-3899-4b43-9f15-3fa3f669e1e8 kind: Event lastTimestamp: "2026-07-08T16:45:26Z" message: InferenceService [success-200-isvc-8ceba] is Ready metadata: creationTimestamp: "2026-07-08T16:45:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:45:26Z" name: success-200-isvc-8ceba.18c05e4a9cac1156 namespace: kserve-ci-e2e-test resourceVersion: "16115" uid: bfb667f0-6b1a-422e-80e7-9553fa5e94c8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T17:05:29.834166Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-91fdc-predictor-768bfc64c4-lvnl9 namespace: kserve-ci-e2e-test resourceVersion: "26241" uid: 2eb571fa-5d32-4a82-a0c2-34d9c659a80e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-91fdc-predictor-768bfc64c4-lvnl9 to ip-10-0-132-182.ec2.internal metadata: creationTimestamp: "2026-07-08T17:05:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T17:05:29Z" name: success-200-isvc-91fdc-predictor-768bfc64c4-lvnl9.18c05f62abe6cb4f namespace: kserve-ci-e2e-test resourceVersion: "26246" uid: ec3e2068-a4fe-4c49-b8af-c0939896b084 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-91fdc-predictor-768bfc64c4-lvnl9 namespace: kserve-ci-e2e-test resourceVersion: "26251" uid: 2eb571fa-5d32-4a82-a0c2-34d9c659a80e kind: Event lastTimestamp: "2026-07-08T17:05:30Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T17:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T17:05:30Z" name: success-200-isvc-91fdc-predictor-768bfc64c4-lvnl9.18c05f62c5dc1092 namespace: kserve-ci-e2e-test resourceVersion: "26293" uid: 7b11ce7b-174f-4bfb-991f-42ed9a1c1023 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-91fdc-predictor-768bfc64c4-lvnl9 namespace: kserve-ci-e2e-test resourceVersion: "26242" uid: 2eb571fa-5d32-4a82-a0c2-34d9c659a80e kind: Event lastTimestamp: "2026-07-08T17:05:30Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1707" already present on machine metadata: creationTimestamp: "2026-07-08T17:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:30Z" name: success-200-isvc-91fdc-predictor-768bfc64c4-lvnl9.18c05f62c7340200 namespace: kserve-ci-e2e-test resourceVersion: "26295" uid: c995f2b9-0997-4049-9050-91782c0ffbb8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-91fdc-predictor-768bfc64c4-lvnl9 namespace: kserve-ci-e2e-test resourceVersion: "26242" uid: 2eb571fa-5d32-4a82-a0c2-34d9c659a80e kind: Event lastTimestamp: "2026-07-08T17:05:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T17:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:30Z" name: success-200-isvc-91fdc-predictor-768bfc64c4-lvnl9.18c05f62ca185ddf namespace: kserve-ci-e2e-test resourceVersion: "26297" uid: afd26024-31c8-4fb7-9282-f5181039f196 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-91fdc-predictor-768bfc64c4-lvnl9 namespace: kserve-ci-e2e-test resourceVersion: "26242" uid: 2eb571fa-5d32-4a82-a0c2-34d9c659a80e kind: Event lastTimestamp: "2026-07-08T17:05:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T17:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:30Z" name: success-200-isvc-91fdc-predictor-768bfc64c4-lvnl9.18c05f62ca892b18 namespace: kserve-ci-e2e-test resourceVersion: "26298" uid: c8f2748f-62b7-47b0-ad54-8ca712ab28d0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-91fdc-predictor-768bfc64c4-lvnl9 namespace: kserve-ci-e2e-test resourceVersion: "26242" uid: 2eb571fa-5d32-4a82-a0c2-34d9c659a80e kind: Event lastTimestamp: "2026-07-08T17:05:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T17:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:30Z" name: success-200-isvc-91fdc-predictor-768bfc64c4-lvnl9.18c05f62ca92b640 namespace: kserve-ci-e2e-test resourceVersion: "26299" uid: e841ed6e-346c-46f1-a53a-2425ea0d458a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-91fdc-predictor-768bfc64c4-lvnl9 namespace: kserve-ci-e2e-test resourceVersion: "26242" uid: 2eb571fa-5d32-4a82-a0c2-34d9c659a80e kind: Event lastTimestamp: "2026-07-08T17:05:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T17:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:30Z" name: success-200-isvc-91fdc-predictor-768bfc64c4-lvnl9.18c05f62cd91a6a5 namespace: kserve-ci-e2e-test resourceVersion: "26303" uid: de16e75a-413b-4e21-a795-607b8af4773e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-91fdc-predictor-768bfc64c4-lvnl9 namespace: kserve-ci-e2e-test resourceVersion: "26242" uid: 2eb571fa-5d32-4a82-a0c2-34d9c659a80e kind: Event lastTimestamp: "2026-07-08T17:05:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:30Z" name: success-200-isvc-91fdc-predictor-768bfc64c4-lvnl9.18c05f62ce02a2b0 namespace: kserve-ci-e2e-test resourceVersion: "26304" uid: ee837fa3-ecd7-4f7a-b50f-54899ecf843b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T17:05:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-91fdc-predictor-768bfc64c4-lvnl9 namespace: kserve-ci-e2e-test resourceVersion: "26242" uid: 2eb571fa-5d32-4a82-a0c2-34d9c659a80e kind: Event lastTimestamp: "2026-07-08T17:06:08Z" message: 'Readiness probe failed: dial tcp 10.132.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:05:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:06:08Z" name: success-200-isvc-91fdc-predictor-768bfc64c4-lvnl9.18c05f633ad18cfd namespace: kserve-ci-e2e-test resourceVersion: "26684" uid: 159b767f-8245-4a25-93a3-7b4237df5f08 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:14:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-91fdc-predictor-768bfc64c4-lvnl9 namespace: kserve-ci-e2e-test resourceVersion: "26242" uid: 2eb571fa-5d32-4a82-a0c2-34d9c659a80e kind: Event lastTimestamp: "2026-07-08T17:14:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T17:14:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:14:44Z" name: success-200-isvc-91fdc-predictor-768bfc64c4-lvnl9.18c05fe3da33f9a4 namespace: kserve-ci-e2e-test resourceVersion: "30355" uid: cd3bcb04-f38a-4a0a-830b-d9d56c100630 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:14:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-91fdc-predictor-768bfc64c4-lvnl9 namespace: kserve-ci-e2e-test resourceVersion: "26242" uid: 2eb571fa-5d32-4a82-a0c2-34d9c659a80e kind: Event lastTimestamp: "2026-07-08T17:14:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:14:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:14:44Z" name: success-200-isvc-91fdc-predictor-768bfc64c4-lvnl9.18c05fe3da347ba5 namespace: kserve-ci-e2e-test resourceVersion: "30356" uid: 4033174c-928a-4ac4-8f37-17ae3c62676e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-91fdc-predictor-768bfc64c4 namespace: kserve-ci-e2e-test resourceVersion: "26229" uid: 29eb4396-e442-4838-a950-864d098bc1f8 kind: Event lastTimestamp: "2026-07-08T17:05:29Z" message: 'Created pod: success-200-isvc-91fdc-predictor-768bfc64c4-lvnl9' metadata: creationTimestamp: "2026-07-08T17:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:05:29Z" name: success-200-isvc-91fdc-predictor-768bfc64c4.18c05f62ab8b94d3 namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 7e7aa122-612e-47da-9b70-a74d17f3c018 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-91fdc-predictor namespace: kserve-ci-e2e-test resourceVersion: "26221" uid: cc75e873-8620-4e32-bc60-a7d34371f681 kind: Event lastTimestamp: "2026-07-08T17:05:29Z" message: Scaled up replica set success-200-isvc-91fdc-predictor-768bfc64c4 from 0 to 1 metadata: creationTimestamp: "2026-07-08T17:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:05:29Z" name: success-200-isvc-91fdc-predictor.18c05f62a9bc877f namespace: kserve-ci-e2e-test resourceVersion: "26232" uid: 8be1b726-2f89-449a-a241-f47d6d90fbeb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-91fdc-predictor namespace: kserve-ci-e2e-test resourceVersion: "26228" uid: d2ca9011-4ec7-453d-a287-72f5e97f7624 kind: Event lastTimestamp: "2026-07-08T17:05:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:05:44Z" name: success-200-isvc-91fdc-predictor.18c05f66284132d5 namespace: kserve-ci-e2e-test resourceVersion: "26438" uid: ea7872c3-946e-4f36-95a7-8bd2c2172489 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-91fdc-predictor namespace: kserve-ci-e2e-test resourceVersion: "26228" uid: d2ca9011-4ec7-453d-a287-72f5e97f7624 kind: Event lastTimestamp: "2026-07-08T17:05:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:05:44Z" name: success-200-isvc-91fdc-predictor.18c05f6628b68dc7 namespace: kserve-ci-e2e-test resourceVersion: "26440" uid: 9382e219-7e35-4710-803c-33d55a036ae6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T17:05:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-91fdc-predictor namespace: kserve-ci-e2e-test resourceVersion: "26439" uid: d2ca9011-4ec7-453d-a287-72f5e97f7624 kind: Event lastTimestamp: "2026-07-08T17:06:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:06:59Z" name: success-200-isvc-91fdc-predictor.18c05f69a77bd8fb namespace: kserve-ci-e2e-test resourceVersion: "27121" uid: 58ff93c1-087f-4beb-88c3-5464036c3d8a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T17:05:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-91fdc-predictor namespace: kserve-ci-e2e-test resourceVersion: "26439" uid: d2ca9011-4ec7-453d-a287-72f5e97f7624 kind: Event lastTimestamp: "2026-07-08T17:06:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:06:59Z" name: success-200-isvc-91fdc-predictor.18c05f69a7f40dc8 namespace: kserve-ci-e2e-test resourceVersion: "27122" uid: ab9487ae-c2e8-49e7-a380-d9db182428b1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T17:05:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-91fdc namespace: kserve-ci-e2e-test resourceVersion: "26194" uid: 672ee687-473d-4df9-a478-02728560231d kind: Event lastTimestamp: "2026-07-08T17:06:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T17:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:06:18Z" name: success-200-isvc-91fdc.18c05f62a9ec9d18 namespace: kserve-ci-e2e-test resourceVersion: "26783" uid: 5a207737-687d-4806-8f40-c36c055d8a4b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-91fdc namespace: kserve-ci-e2e-test resourceVersion: "26194" uid: 672ee687-473d-4df9-a478-02728560231d kind: Event lastTimestamp: "2026-07-08T17:05:29Z" message: 'Failed to update status for InferenceService "success-200-isvc-91fdc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-91fdc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T17:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:05:29Z" name: success-200-isvc-91fdc.18c05f62aa4d50b8 namespace: kserve-ci-e2e-test resourceVersion: "26236" uid: b9eb1b2c-ab7b-4acd-8e1d-e90082997e45 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-91fdc namespace: kserve-ci-e2e-test resourceVersion: "26194" uid: 672ee687-473d-4df9-a478-02728560231d kind: Event lastTimestamp: "2026-07-08T17:05:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-91fdc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T17:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:05:29Z" name: success-200-isvc-91fdc.18c05f62aa4d90f4 namespace: kserve-ci-e2e-test resourceVersion: "26238" uid: 6c0c2a4b-c908-4090-b1d8-610f71170fe2 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:06:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-91fdc namespace: kserve-ci-e2e-test resourceVersion: "26781" uid: 672ee687-473d-4df9-a478-02728560231d kind: Event lastTimestamp: "2026-07-08T17:06:18Z" message: InferenceService [success-200-isvc-91fdc] is Ready metadata: creationTimestamp: "2026-07-08T17:06:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:06:18Z" name: success-200-isvc-91fdc.18c05f6df6b05ff1 namespace: kserve-ci-e2e-test resourceVersion: "26782" uid: eebfde03-b59a-4f57-926d-d688bca5cc5a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:42:49.272739Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ed4fa-predictor-774674684-75tb6 namespace: kserve-ci-e2e-test resourceVersion: "14192" uid: 8ec8f670-0437-4ba1-89e6-9647e0f54703 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ed4fa-predictor-774674684-75tb6 to ip-10-0-132-182.ec2.internal metadata: creationTimestamp: "2026-07-08T16:42:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:42:49Z" name: success-200-isvc-ed4fa-predictor-774674684-75tb6.18c05e25e41ee30d namespace: kserve-ci-e2e-test resourceVersion: "14197" uid: 0eb0e876-7cd0-44e1-b362-0dc9e378f3d9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:49Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ed4fa-predictor-774674684-75tb6 namespace: kserve-ci-e2e-test resourceVersion: "14204" uid: 8ec8f670-0437-4ba1-89e6-9647e0f54703 kind: Event lastTimestamp: "2026-07-08T16:42:49Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:42:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:42:49Z" name: success-200-isvc-ed4fa-predictor-774674684-75tb6.18c05e25ff2f2681 namespace: kserve-ci-e2e-test resourceVersion: "14285" uid: 9c08d85e-c465-460e-b0bd-6a465c8101f3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ed4fa-predictor-774674684-75tb6 namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: 8ec8f670-0437-4ba1-89e6-9647e0f54703 kind: Event lastTimestamp: "2026-07-08T16:42:49Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1707" metadata: creationTimestamp: "2026-07-08T16:42:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:42:49Z" name: success-200-isvc-ed4fa-predictor-774674684-75tb6.18c05e2601c2b8df namespace: kserve-ci-e2e-test resourceVersion: "14290" uid: ed39c12d-7ebe-4fdf-a109-a6786b47ecaf reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ed4fa-predictor-774674684-75tb6 namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: 8ec8f670-0437-4ba1-89e6-9647e0f54703 kind: Event lastTimestamp: "2026-07-08T16:43:03Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1707" in 13.56s (13.56s including waiting). Image size: 1335613048 bytes.' metadata: creationTimestamp: "2026-07-08T16:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:03Z" name: success-200-isvc-ed4fa-predictor-774674684-75tb6.18c05e292a069314 namespace: kserve-ci-e2e-test resourceVersion: "14449" uid: 5d6f0dd6-f9a5-4191-b029-74a69b18c11e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ed4fa-predictor-774674684-75tb6 namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: 8ec8f670-0437-4ba1-89e6-9647e0f54703 kind: Event lastTimestamp: "2026-07-08T16:43:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:03Z" name: success-200-isvc-ed4fa-predictor-774674684-75tb6.18c05e292d4d9863 namespace: kserve-ci-e2e-test resourceVersion: "14450" uid: 2b601502-b5ee-4f61-805e-b168c6a7537c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ed4fa-predictor-774674684-75tb6 namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: 8ec8f670-0437-4ba1-89e6-9647e0f54703 kind: Event lastTimestamp: "2026-07-08T16:43:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:03Z" name: success-200-isvc-ed4fa-predictor-774674684-75tb6.18c05e292ddc1311 namespace: kserve-ci-e2e-test resourceVersion: "14451" uid: e672bf27-dbd7-49ce-b88d-92165f72a328 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ed4fa-predictor-774674684-75tb6 namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: 8ec8f670-0437-4ba1-89e6-9647e0f54703 kind: Event lastTimestamp: "2026-07-08T16:43:03Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-08T16:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:03Z" name: success-200-isvc-ed4fa-predictor-774674684-75tb6.18c05e292deb2ae5 namespace: kserve-ci-e2e-test resourceVersion: "14452" uid: c69006dd-d286-4a67-8d72-a3aee34026f6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ed4fa-predictor-774674684-75tb6 namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: 8ec8f670-0437-4ba1-89e6-9647e0f54703 kind: Event lastTimestamp: "2026-07-08T16:43:05Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.17s (2.17s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-08T16:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:05Z" name: success-200-isvc-ed4fa-predictor-774674684-75tb6.18c05e29af45b99b namespace: kserve-ci-e2e-test resourceVersion: "14487" uid: 20e60248-638d-4f74-90e0-67ea2abdaa57 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ed4fa-predictor-774674684-75tb6 namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: 8ec8f670-0437-4ba1-89e6-9647e0f54703 kind: Event lastTimestamp: "2026-07-08T16:43:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:05Z" name: success-200-isvc-ed4fa-predictor-774674684-75tb6.18c05e29b26e627b namespace: kserve-ci-e2e-test resourceVersion: "14488" uid: 38936b91-53c6-4da6-8179-db0d101bf9dc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ed4fa-predictor-774674684-75tb6 namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: 8ec8f670-0437-4ba1-89e6-9647e0f54703 kind: Event lastTimestamp: "2026-07-08T16:43:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:05Z" name: success-200-isvc-ed4fa-predictor-774674684-75tb6.18c05e29b2de4827 namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: dcfc0292-2a20-49d4-b03d-6b3b6daf0d9f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:43:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ed4fa-predictor-774674684-75tb6 namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: 8ec8f670-0437-4ba1-89e6-9647e0f54703 kind: Event lastTimestamp: "2026-07-08T16:43:42Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:43:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:43:42Z" name: success-200-isvc-ed4fa-predictor-774674684-75tb6.18c05e29e584acc1 namespace: kserve-ci-e2e-test resourceVersion: "14821" uid: 619aeb69-67da-4dd5-850e-311ef32d5ee0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ed4fa-predictor-774674684-75tb6 namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: 8ec8f670-0437-4ba1-89e6-9647e0f54703 kind: Event lastTimestamp: "2026-07-08T16:44:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:39Z" name: success-200-isvc-ed4fa-predictor-774674684-75tb6.18c05e3f864309a4 namespace: kserve-ci-e2e-test resourceVersion: "15390" uid: c742512a-eb41-40dd-839b-71ea9813a178 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ed4fa-predictor-774674684-75tb6 namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: 8ec8f670-0437-4ba1-89e6-9647e0f54703 kind: Event lastTimestamp: "2026-07-08T16:44:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:39Z" name: success-200-isvc-ed4fa-predictor-774674684-75tb6.18c05e3f86439b55 namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: cfe7e6ca-19f1-47db-87c3-87f514e24aff reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ed4fa-predictor-774674684-75tb6 namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: 8ec8f670-0437-4ba1-89e6-9647e0f54703 kind: Event lastTimestamp: "2026-07-08T16:44:43Z" message: 'Readiness probe failed: Get "https://10.132.0.24:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-07-08T16:44:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:43Z" name: success-200-isvc-ed4fa-predictor-774674684-75tb6.18c05e407b5f1734 namespace: kserve-ci-e2e-test resourceVersion: "15536" uid: cada9c65-654b-495b-9ea5-8c95e652aa4a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ed4fa-predictor-774674684-75tb6 namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: 8ec8f670-0437-4ba1-89e6-9647e0f54703 kind: Event lastTimestamp: "2026-07-08T16:44:43Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: i/o timeout' metadata: creationTimestamp: "2026-07-08T16:44:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:43Z" name: success-200-isvc-ed4fa-predictor-774674684-75tb6.18c05e407b9f3360 namespace: kserve-ci-e2e-test resourceVersion: "15537" uid: d318ba63-c68d-4c7a-8b1d-73a684487752 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ed4fa-predictor-774674684 namespace: kserve-ci-e2e-test resourceVersion: "14185" uid: 62351fae-0357-4135-a9d9-86adc3f5208a kind: Event lastTimestamp: "2026-07-08T16:42:49Z" message: 'Created pod: success-200-isvc-ed4fa-predictor-774674684-75tb6' metadata: creationTimestamp: "2026-07-08T16:42:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:42:49Z" name: success-200-isvc-ed4fa-predictor-774674684.18c05e25e3be1b40 namespace: kserve-ci-e2e-test resourceVersion: "14194" uid: 4ea6c450-ca4a-441d-8508-b6f281456cac reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:42:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ed4fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "14183" uid: d607f908-d2d9-4376-939c-fb72284c883c kind: Event lastTimestamp: "2026-07-08T16:42:49Z" message: Scaled up replica set success-200-isvc-ed4fa-predictor-774674684 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:42:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:42:49Z" name: success-200-isvc-ed4fa-predictor.18c05e25e10eb26d namespace: kserve-ci-e2e-test resourceVersion: "14188" uid: 8370d049-3b9d-4e7a-a61b-a2e0c04c75a0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:43:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ed4fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "14184" uid: eca002dc-7b01-4675-9478-b100903aee96 kind: Event lastTimestamp: "2026-07-08T16:43:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:43:49Z" name: success-200-isvc-ed4fa-predictor.18c05e295fa56127 namespace: kserve-ci-e2e-test resourceVersion: "14871" uid: 6c4c297a-5136-457c-b613-a6df7e247754 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:43:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ed4fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "14184" uid: eca002dc-7b01-4675-9478-b100903aee96 kind: Event lastTimestamp: "2026-07-08T16:43:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:43:49Z" name: success-200-isvc-ed4fa-predictor.18c05e296019a97e namespace: kserve-ci-e2e-test resourceVersion: "14872" uid: db614ef0-7b21-46bb-9577-9d1ba0ec69c7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:44:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ed4fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "14465" uid: eca002dc-7b01-4675-9478-b100903aee96 kind: Event lastTimestamp: "2026-07-08T16:44:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:44:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:44:34Z" name: success-200-isvc-ed4fa-predictor.18c05e375b457bc9 namespace: kserve-ci-e2e-test resourceVersion: "15295" uid: 986913dc-b1c3-4646-af87-155e8249f189 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:44:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ed4fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "14465" uid: eca002dc-7b01-4675-9478-b100903aee96 kind: Event lastTimestamp: "2026-07-08T16:44:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:44:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:44:34Z" name: success-200-isvc-ed4fa-predictor.18c05e375bbc0f18 namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: d85458e2-c1f5-442e-a976-ad6ad176db93 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:42:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ed4fa namespace: kserve-ci-e2e-test resourceVersion: "14176" uid: 96c3a682-d839-4e0e-b812-7152679e6a5d kind: Event lastTimestamp: "2026-07-08T16:43:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:42:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:43:52Z" name: success-200-isvc-ed4fa.18c05e25e116d1eb namespace: kserve-ci-e2e-test resourceVersion: "14914" uid: 11202ef1-189e-408a-98a0-1b868c65eb24 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:43:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ed4fa namespace: kserve-ci-e2e-test resourceVersion: "14912" uid: 96c3a682-d839-4e0e-b812-7152679e6a5d kind: Event lastTimestamp: "2026-07-08T16:43:52Z" message: InferenceService [success-200-isvc-ed4fa] is Ready metadata: creationTimestamp: "2026-07-08T16:43:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:43:52Z" name: success-200-isvc-ed4fa.18c05e34a0a8dc88 namespace: kserve-ci-e2e-test resourceVersion: "14913" uid: 5ce6d198-2873-4510-bc4b-a7390394cf67 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:55:44.485943Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f6dd6-predictor-6df86f94d4-vk2v4 namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: ee220760-6390-446a-b641-f2bfce2f1c52 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f6dd6-predictor-6df86f94d4-vk2v4 to ip-10-0-132-182.ec2.internal metadata: creationTimestamp: "2026-07-08T16:55:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:55:44Z" name: success-200-isvc-f6dd6-predictor-6df86f94d4-vk2v4.18c05eda626da5ab namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: fec985cd-bdd5-484d-80ae-105689705d5a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:44Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f6dd6-predictor-6df86f94d4-vk2v4 namespace: kserve-ci-e2e-test resourceVersion: "21236" uid: ee220760-6390-446a-b641-f2bfce2f1c52 kind: Event lastTimestamp: "2026-07-08T16:55:44Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:55:44Z" name: success-200-isvc-f6dd6-predictor-6df86f94d4-vk2v4.18c05eda7c2cc989 namespace: kserve-ci-e2e-test resourceVersion: "21274" uid: 770a877c-ce6c-46c3-bf88-6c63896188cc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f6dd6-predictor-6df86f94d4-vk2v4 namespace: kserve-ci-e2e-test resourceVersion: "21227" uid: ee220760-6390-446a-b641-f2bfce2f1c52 kind: Event lastTimestamp: "2026-07-08T16:55:44Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1707" already present on machine metadata: creationTimestamp: "2026-07-08T16:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:44Z" name: success-200-isvc-f6dd6-predictor-6df86f94d4-vk2v4.18c05eda7d9082f6 namespace: kserve-ci-e2e-test resourceVersion: "21276" uid: 99c670cd-a8d6-4ccb-9b00-c670d63b1659 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f6dd6-predictor-6df86f94d4-vk2v4 namespace: kserve-ci-e2e-test resourceVersion: "21227" uid: ee220760-6390-446a-b641-f2bfce2f1c52 kind: Event lastTimestamp: "2026-07-08T16:55:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:44Z" name: success-200-isvc-f6dd6-predictor-6df86f94d4-vk2v4.18c05eda80cde7b4 namespace: kserve-ci-e2e-test resourceVersion: "21279" uid: 508e4257-8389-4a1a-9b33-5e171119d88b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f6dd6-predictor-6df86f94d4-vk2v4 namespace: kserve-ci-e2e-test resourceVersion: "21227" uid: ee220760-6390-446a-b641-f2bfce2f1c52 kind: Event lastTimestamp: "2026-07-08T16:55:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:45Z" name: success-200-isvc-f6dd6-predictor-6df86f94d4-vk2v4.18c05eda813ad9ee namespace: kserve-ci-e2e-test resourceVersion: "21280" uid: 07e97ac5-e6ca-4f63-9278-3ae55533a74d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f6dd6-predictor-6df86f94d4-vk2v4 namespace: kserve-ci-e2e-test resourceVersion: "21227" uid: ee220760-6390-446a-b641-f2bfce2f1c52 kind: Event lastTimestamp: "2026-07-08T16:55:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:45Z" name: success-200-isvc-f6dd6-predictor-6df86f94d4-vk2v4.18c05eda81442e80 namespace: kserve-ci-e2e-test resourceVersion: "21281" uid: 83c20cda-74da-4f42-bd17-ccbd38b051bf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f6dd6-predictor-6df86f94d4-vk2v4 namespace: kserve-ci-e2e-test resourceVersion: "21227" uid: ee220760-6390-446a-b641-f2bfce2f1c52 kind: Event lastTimestamp: "2026-07-08T16:55:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:45Z" name: success-200-isvc-f6dd6-predictor-6df86f94d4-vk2v4.18c05eda83e20bf9 namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: 24bcd2e8-29a9-424f-b7c0-5001127f86ce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f6dd6-predictor-6df86f94d4-vk2v4 namespace: kserve-ci-e2e-test resourceVersion: "21227" uid: ee220760-6390-446a-b641-f2bfce2f1c52 kind: Event lastTimestamp: "2026-07-08T16:55:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:45Z" name: success-200-isvc-f6dd6-predictor-6df86f94d4-vk2v4.18c05eda8451f672 namespace: kserve-ci-e2e-test resourceVersion: "21283" uid: 1d255f71-49d5-49a2-b5ee-6ff0d6ebc4a4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:55:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f6dd6-predictor-6df86f94d4-vk2v4 namespace: kserve-ci-e2e-test resourceVersion: "21227" uid: ee220760-6390-446a-b641-f2bfce2f1c52 kind: Event lastTimestamp: "2026-07-08T16:56:22Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:55:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:56:22Z" name: success-200-isvc-f6dd6-predictor-6df86f94d4-vk2v4.18c05edae6ef4e90 namespace: kserve-ci-e2e-test resourceVersion: "21677" uid: a4e1e315-a9c1-4025-97d5-0184ef2ecd39 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f6dd6-predictor-6df86f94d4-vk2v4 namespace: kserve-ci-e2e-test resourceVersion: "21227" uid: ee220760-6390-446a-b641-f2bfce2f1c52 kind: Event lastTimestamp: "2026-07-08T17:04:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:59Z" name: success-200-isvc-f6dd6-predictor-6df86f94d4-vk2v4.18c05f5b953e47bb namespace: kserve-ci-e2e-test resourceVersion: "25765" uid: f1e779ae-8c85-4b85-a929-384afe050710 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f6dd6-predictor-6df86f94d4-vk2v4 namespace: kserve-ci-e2e-test resourceVersion: "21227" uid: ee220760-6390-446a-b641-f2bfce2f1c52 kind: Event lastTimestamp: "2026-07-08T17:04:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:04:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:04:59Z" name: success-200-isvc-f6dd6-predictor-6df86f94d4-vk2v4.18c05f5b953ec21f namespace: kserve-ci-e2e-test resourceVersion: "25768" uid: 0c4acd2a-cf64-45b8-9030-29083ee31a16 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f6dd6-predictor-6df86f94d4 namespace: kserve-ci-e2e-test resourceVersion: "21207" uid: 29cbb217-a7f8-499f-bc89-ad9a993e4508 kind: Event lastTimestamp: "2026-07-08T16:55:44Z" message: 'Created pod: success-200-isvc-f6dd6-predictor-6df86f94d4-vk2v4' metadata: creationTimestamp: "2026-07-08T16:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:44Z" name: success-200-isvc-f6dd6-predictor-6df86f94d4.18c05eda61d70a54 namespace: kserve-ci-e2e-test resourceVersion: "21226" uid: 852a7052-13a3-435b-b268-1237ddbf1a4b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f6dd6-predictor namespace: kserve-ci-e2e-test resourceVersion: "21199" uid: 98b169c9-bbee-4581-88e3-2b53ca0b1848 kind: Event lastTimestamp: "2026-07-08T16:55:44Z" message: Scaled up replica set success-200-isvc-f6dd6-predictor-6df86f94d4 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:44Z" name: success-200-isvc-f6dd6-predictor.18c05eda5f6f7386 namespace: kserve-ci-e2e-test resourceVersion: "21211" uid: 4fc3b744-5200-43bc-885f-ff5e3ad895ca reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:55:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f6dd6-predictor namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: 8a486601-b292-44e0-82ff-11dfba1f73b1 kind: Event lastTimestamp: "2026-07-08T16:56:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:56:14Z" name: success-200-isvc-f6dd6-predictor.18c05eddddd18e3e namespace: kserve-ci-e2e-test resourceVersion: "21604" uid: 162e0514-82c8-4b21-8995-a8a956c1b64e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:55:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f6dd6-predictor namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: 8a486601-b292-44e0-82ff-11dfba1f73b1 kind: Event lastTimestamp: "2026-07-08T16:56:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:56:14Z" name: success-200-isvc-f6dd6-predictor.18c05eddde4a8ffb namespace: kserve-ci-e2e-test resourceVersion: "21605" uid: d0880b72-0ae6-4330-a0c4-5c163b6df69d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:56:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f6dd6-predictor namespace: kserve-ci-e2e-test resourceVersion: "21444" uid: 8a486601-b292-44e0-82ff-11dfba1f73b1 kind: Event lastTimestamp: "2026-07-08T16:56:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:56:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:56:59Z" name: success-200-isvc-f6dd6-predictor.18c05ee4dc31f466 namespace: kserve-ci-e2e-test resourceVersion: "22004" uid: 3f6d8777-621f-48dc-adf2-5504abcd2b13 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:56:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f6dd6-predictor namespace: kserve-ci-e2e-test resourceVersion: "21444" uid: 8a486601-b292-44e0-82ff-11dfba1f73b1 kind: Event lastTimestamp: "2026-07-08T16:56:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:56:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:56:59Z" name: success-200-isvc-f6dd6-predictor.18c05ee4dcb2ce60 namespace: kserve-ci-e2e-test resourceVersion: "22005" uid: 0c9cf8a0-aec7-4e22-b13c-3a0ecbe35a00 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:55:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f6dd6 namespace: kserve-ci-e2e-test resourceVersion: "21178" uid: 7bd74f36-01c7-4206-b445-3d480b7c5e31 kind: Event lastTimestamp: "2026-07-08T16:56:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:56:32Z" name: success-200-isvc-f6dd6.18c05eda5f758d17 namespace: kserve-ci-e2e-test resourceVersion: "21761" uid: 46f3eae9-921a-4ac4-b10d-606645be01f0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f6dd6 namespace: kserve-ci-e2e-test resourceVersion: "21178" uid: 7bd74f36-01c7-4206-b445-3d480b7c5e31 kind: Event lastTimestamp: "2026-07-08T16:55:44Z" message: 'Failed to update status for InferenceService "success-200-isvc-f6dd6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f6dd6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:55:44Z" name: success-200-isvc-f6dd6.18c05eda602b2053 namespace: kserve-ci-e2e-test resourceVersion: "21218" uid: 5be0ab23-ead2-4b07-9a7d-4b3661ee23ea reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f6dd6 namespace: kserve-ci-e2e-test resourceVersion: "21178" uid: 7bd74f36-01c7-4206-b445-3d480b7c5e31 kind: Event lastTimestamp: "2026-07-08T16:55:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f6dd6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:55:44Z" name: success-200-isvc-f6dd6.18c05eda602b7073 namespace: kserve-ci-e2e-test resourceVersion: "21219" uid: 033876ad-cb90-4128-bb03-034e3a68460d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f6dd6 namespace: kserve-ci-e2e-test resourceVersion: "21759" uid: 7bd74f36-01c7-4206-b445-3d480b7c5e31 kind: Event lastTimestamp: "2026-07-08T16:56:32Z" message: InferenceService [success-200-isvc-f6dd6] is Ready metadata: creationTimestamp: "2026-07-08T16:56:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:56:32Z" name: success-200-isvc-f6dd6.18c05ee5a2877ca3 namespace: kserve-ci-e2e-test resourceVersion: "21760" uid: 6402c0f2-9901-4656-a049-9b5e12ca6cca reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:45:19.380949Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fabf2-predictor-7998766446-tv7kw namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: 75e58397-959f-43bb-9709-4f1b09463274 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-fabf2-predictor-7998766446-tv7kw to ip-10-0-132-182.ec2.internal metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:45:19Z" name: success-200-isvc-fabf2-predictor-7998766446-tv7kw.18c05e48d744688d namespace: kserve-ci-e2e-test resourceVersion: "15941" uid: a4b8f575-c02b-4fcd-9fa5-700bf6338d28 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:19Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fabf2-predictor-7998766446-tv7kw namespace: kserve-ci-e2e-test resourceVersion: "15958" uid: 75e58397-959f-43bb-9709-4f1b09463274 kind: Event lastTimestamp: "2026-07-08T16:45:19Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:45:19Z" name: success-200-isvc-fabf2-predictor-7998766446-tv7kw.18c05e48f096fe29 namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: 7c562d61-28aa-49be-8433-63bcce7ad2dd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fabf2-predictor-7998766446-tv7kw namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: 75e58397-959f-43bb-9709-4f1b09463274 kind: Event lastTimestamp: "2026-07-08T16:45:19Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1707" already present on machine metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:19Z" name: success-200-isvc-fabf2-predictor-7998766446-tv7kw.18c05e48f1f86f52 namespace: kserve-ci-e2e-test resourceVersion: "16006" uid: 4b3b8bee-b853-429f-b3b7-73d00e2c16c9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fabf2-predictor-7998766446-tv7kw namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: 75e58397-959f-43bb-9709-4f1b09463274 kind: Event lastTimestamp: "2026-07-08T16:45:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:19Z" name: success-200-isvc-fabf2-predictor-7998766446-tv7kw.18c05e48f4a7944c namespace: kserve-ci-e2e-test resourceVersion: "16007" uid: 5274a162-c8cd-4000-9644-cf4932c0a438 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fabf2-predictor-7998766446-tv7kw namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: 75e58397-959f-43bb-9709-4f1b09463274 kind: Event lastTimestamp: "2026-07-08T16:45:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:19Z" name: success-200-isvc-fabf2-predictor-7998766446-tv7kw.18c05e48f5049d39 namespace: kserve-ci-e2e-test resourceVersion: "16008" uid: c51bd159-3ab5-4ef3-8a27-b909c0c1d3a1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fabf2-predictor-7998766446-tv7kw namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: 75e58397-959f-43bb-9709-4f1b09463274 kind: Event lastTimestamp: "2026-07-08T16:45:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:19Z" name: success-200-isvc-fabf2-predictor-7998766446-tv7kw.18c05e48f50d9eef namespace: kserve-ci-e2e-test resourceVersion: "16009" uid: 9f29a71a-61bb-4fb1-935b-2ee8fbf5f53c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fabf2-predictor-7998766446-tv7kw namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: 75e58397-959f-43bb-9709-4f1b09463274 kind: Event lastTimestamp: "2026-07-08T16:45:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:19Z" name: success-200-isvc-fabf2-predictor-7998766446-tv7kw.18c05e48f8b6e19d namespace: kserve-ci-e2e-test resourceVersion: "16011" uid: 43571fdf-e933-49bc-b0c2-57e402e903cb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fabf2-predictor-7998766446-tv7kw namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: 75e58397-959f-43bb-9709-4f1b09463274 kind: Event lastTimestamp: "2026-07-08T16:45:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:19Z" name: success-200-isvc-fabf2-predictor-7998766446-tv7kw.18c05e48f9192070 namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: 7b7068d5-58db-4c9e-9302-8efd0768626a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-08T16:45:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fabf2-predictor-7998766446-tv7kw namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: 75e58397-959f-43bb-9709-4f1b09463274 kind: Event lastTimestamp: "2026-07-08T16:54:37Z" message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:37Z" name: success-200-isvc-fabf2-predictor-7998766446-tv7kw.18c05e4939d12a8f namespace: kserve-ci-e2e-test resourceVersion: "20344" uid: e439c2e6-1150-4b58-b6a1-0cc2662ced7e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fabf2-predictor-7998766446-tv7kw namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: 75e58397-959f-43bb-9709-4f1b09463274 kind: Event lastTimestamp: "2026-07-08T16:54:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:34Z" name: success-200-isvc-fabf2-predictor-7998766446-tv7kw.18c05eca0180b594 namespace: kserve-ci-e2e-test resourceVersion: "20230" uid: 67cd5071-6c61-4b0e-8177-3ba308aea9e8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fabf2-predictor-7998766446-tv7kw namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: 75e58397-959f-43bb-9709-4f1b09463274 kind: Event lastTimestamp: "2026-07-08T16:54:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:34Z" name: success-200-isvc-fabf2-predictor-7998766446-tv7kw.18c05eca01820e1a namespace: kserve-ci-e2e-test resourceVersion: "20234" uid: a8bc0e4a-8832-4859-af9a-885e3fb92b41 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fabf2-predictor-7998766446-tv7kw namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: 75e58397-959f-43bb-9709-4f1b09463274 kind: Event lastTimestamp: "2026-07-08T16:54:37Z" message: 'Readiness probe failed: Get "https://10.132.0.28:8643/healthz": dial tcp 10.132.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:54:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:37Z" name: success-200-isvc-fabf2-predictor-7998766446-tv7kw.18c05ecaae41f0a9 namespace: kserve-ci-e2e-test resourceVersion: "20343" uid: 471dab35-588e-41e5-b716-fa83ecb6bd24 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-fabf2-predictor-7998766446 namespace: kserve-ci-e2e-test resourceVersion: "15914" uid: aaff3d14-abdc-46ba-a01f-60bc35d95f55 kind: Event lastTimestamp: "2026-07-08T16:45:19Z" message: 'Created pod: success-200-isvc-fabf2-predictor-7998766446-tv7kw' metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:45:19Z" name: success-200-isvc-fabf2-predictor-7998766446.18c05e48d69e5715 namespace: kserve-ci-e2e-test resourceVersion: "15937" uid: d10b668e-8386-4ff3-8f72-bc09336bf22e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-fabf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15911" uid: 13969e30-7331-4106-b7a8-53a236aeb7e4 kind: Event lastTimestamp: "2026-07-08T16:45:19Z" message: Scaled up replica set success-200-isvc-fabf2-predictor-7998766446 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:45:19Z" name: success-200-isvc-fabf2-predictor.18c05e48d408d392 namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: 6e40e673-399b-4861-adaa-a414f542d538 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:45:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fabf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15913" uid: b3e9404c-4478-421e-8632-dcc364731738 kind: Event lastTimestamp: "2026-07-08T16:46:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:04Z" name: success-200-isvc-fabf2-predictor.18c05e4c5294cc0f namespace: kserve-ci-e2e-test resourceVersion: "16450" uid: af88bf1b-0a33-4005-8853-7a7462a08279 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:45:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fabf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15913" uid: b3e9404c-4478-421e-8632-dcc364731738 kind: Event lastTimestamp: "2026-07-08T16:46:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:04Z" name: success-200-isvc-fabf2-predictor.18c05e4c5312416c namespace: kserve-ci-e2e-test resourceVersion: "16451" uid: a0a83583-cecc-4445-9176-580e7ee0f20a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:46:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fabf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16176" uid: b3e9404c-4478-421e-8632-dcc364731738 kind: Event lastTimestamp: "2026-07-08T16:46:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:46:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:49Z" name: success-200-isvc-fabf2-predictor.18c05e56d112e92b namespace: kserve-ci-e2e-test resourceVersion: "16838" uid: 2abaa803-0fb4-4d18-86fb-0d4eb292f7bc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:46:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fabf2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16176" uid: b3e9404c-4478-421e-8632-dcc364731738 kind: Event lastTimestamp: "2026-07-08T16:46:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:46:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:49Z" name: success-200-isvc-fabf2-predictor.18c05e56d185314f namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 7774495e-d100-4679-b1b5-efc290f6999e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:45:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fabf2 namespace: kserve-ci-e2e-test resourceVersion: "15889" uid: c3a5da98-529c-4f2e-a494-da8de8d33751 kind: Event lastTimestamp: "2026-07-08T16:46:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:46:07Z" name: success-200-isvc-fabf2.18c05e48d44d6a16 namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: 5bc7fa18-bcdf-4698-b8b2-b7360bf34afa reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fabf2 namespace: kserve-ci-e2e-test resourceVersion: "16479" uid: c3a5da98-529c-4f2e-a494-da8de8d33751 kind: Event lastTimestamp: "2026-07-08T16:46:07Z" message: InferenceService [success-200-isvc-fabf2] is Ready metadata: creationTimestamp: "2026-07-08T16:46:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:46:07Z" name: success-200-isvc-fabf2.18c05e53f7852c32 namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: ffcb854f-898d-4155-9d71-697b526bbd5f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T17:05:59.534039Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-73f94-65b6995f6-nlths namespace: kserve-ci-e2e-test resourceVersion: "26583" uid: b4bee131-c4b3-45cf-a941-249bf8624a7a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-73f94-65b6995f6-nlths to ip-10-0-132-182.ec2.internal metadata: creationTimestamp: "2026-07-08T17:05:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T17:05:59Z" name: switch-graph-73f94-65b6995f6-nlths.18c05f699626ccbb namespace: kserve-ci-e2e-test resourceVersion: "26588" uid: 1e54f2ea-abba-4135-99c4-39dff615566f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:59Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-73f94-65b6995f6-nlths namespace: kserve-ci-e2e-test resourceVersion: "26592" uid: b4bee131-c4b3-45cf-a941-249bf8624a7a kind: Event lastTimestamp: "2026-07-08T17:05:59Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T17:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T17:05:59Z" name: switch-graph-73f94-65b6995f6-nlths.18c05f69b00df91a namespace: kserve-ci-e2e-test resourceVersion: "26611" uid: c19d9c11-78e9-4200-bc49-ce0ab144fa62 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-73f94} kind: Pod name: switch-graph-73f94-65b6995f6-nlths namespace: kserve-ci-e2e-test resourceVersion: "26585" uid: b4bee131-c4b3-45cf-a941-249bf8624a7a kind: Event lastTimestamp: "2026-07-08T17:05:59Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:86bfaabcc176433317a84dc02e835b9657d32290b963c2e4ae8312aa72bd9d64" already present on machine metadata: creationTimestamp: "2026-07-08T17:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:59Z" name: switch-graph-73f94-65b6995f6-nlths.18c05f69b1800f4d namespace: kserve-ci-e2e-test resourceVersion: "26613" uid: 152ff993-7b5d-4199-9b87-f87ca15216b6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:06:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-73f94} kind: Pod name: switch-graph-73f94-65b6995f6-nlths namespace: kserve-ci-e2e-test resourceVersion: "26585" uid: b4bee131-c4b3-45cf-a941-249bf8624a7a kind: Event lastTimestamp: "2026-07-08T17:06:00Z" message: 'Created container: switch-graph-73f94' metadata: creationTimestamp: "2026-07-08T17:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:06:00Z" name: switch-graph-73f94-65b6995f6-nlths.18c05f69b47c3f44 namespace: kserve-ci-e2e-test resourceVersion: "26614" uid: ba837c5d-1b14-450c-ac06-a89b93c4cc06 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:06:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-73f94} kind: Pod name: switch-graph-73f94-65b6995f6-nlths namespace: kserve-ci-e2e-test resourceVersion: "26585" uid: b4bee131-c4b3-45cf-a941-249bf8624a7a kind: Event lastTimestamp: "2026-07-08T17:06:00Z" message: Started container switch-graph-73f94 metadata: creationTimestamp: "2026-07-08T17:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:06:00Z" name: switch-graph-73f94-65b6995f6-nlths.18c05f69b4ea5231 namespace: kserve-ci-e2e-test resourceVersion: "26615" uid: 0b41f596-fbed-4617-9e7b-3a28c625d5cf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:22:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-73f94} kind: Pod name: switch-graph-73f94-65b6995f6-nlths namespace: kserve-ci-e2e-test resourceVersion: "26585" uid: b4bee131-c4b3-45cf-a941-249bf8624a7a kind: Event lastTimestamp: "2026-07-08T17:22:18Z" message: Stopping container switch-graph-73f94 metadata: creationTimestamp: "2026-07-08T17:22:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:22:18Z" name: switch-graph-73f94-65b6995f6-nlths.18c0604d9358fbbc namespace: kserve-ci-e2e-test resourceVersion: "33478" uid: 225d5961-c27f-41d8-ab5c-6cd71183144a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T17:22:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-73f94} kind: Pod name: switch-graph-73f94-65b6995f6-nlths namespace: kserve-ci-e2e-test resourceVersion: "26585" uid: b4bee131-c4b3-45cf-a941-249bf8624a7a kind: Event lastTimestamp: "2026-07-08T17:22:41Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T17:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:22:41Z" name: switch-graph-73f94-65b6995f6-nlths.18c0604e2fbfa9eb namespace: kserve-ci-e2e-test resourceVersion: "33713" uid: 8a362cfb-6fb2-4e28-a25c-6e41a2ec99e8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-73f94-65b6995f6 namespace: kserve-ci-e2e-test resourceVersion: "26580" uid: e9f059f8-be6b-4274-9917-307f5e55d91e kind: Event lastTimestamp: "2026-07-08T17:05:59Z" message: 'Created pod: switch-graph-73f94-65b6995f6-nlths' metadata: creationTimestamp: "2026-07-08T17:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:05:59Z" name: switch-graph-73f94-65b6995f6.18c05f6995bf7719 namespace: kserve-ci-e2e-test resourceVersion: "26584" uid: 9fea5391-1c0c-40f7-8283-99f17fb38e2a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-73f94 namespace: kserve-ci-e2e-test resourceVersion: "26576" uid: 93fb9722-13e0-4a21-b516-e7cfc9a00801 kind: Event lastTimestamp: "2026-07-08T17:05:59Z" message: Scaled up replica set switch-graph-73f94-65b6995f6 from 0 to 1 metadata: creationTimestamp: "2026-07-08T17:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:05:59Z" name: switch-graph-73f94.18c05f699507a1d0 namespace: kserve-ci-e2e-test resourceVersion: "26581" uid: 726b611c-1be9-471a-91ff-d8df366cd907 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:59Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-73f94 namespace: kserve-ci-e2e-test resourceVersion: "26597" uid: 5e11cb16-dfc4-4d7e-9e57-ae2cac68d786 kind: Event lastTimestamp: "2026-07-08T17:05:59Z" message: InferenceGraph [switch-graph-73f94] is Ready metadata: creationTimestamp: "2026-07-08T17:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:05:59Z" name: switch-graph-73f94.18c05f699ac83b22 namespace: kserve-ci-e2e-test resourceVersion: "26599" uid: 0641acb0-12cd-4569-be9d-f978c4b224ed reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:06:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-73f94 namespace: kserve-ci-e2e-test resourceVersion: "26578" uid: 4dc37f10-f44c-4ff8-b938-5c84a6cdf8d8 kind: Event lastTimestamp: "2026-07-08T17:06:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:06:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:06:29Z" name: switch-graph-73f94.18c05f6d13888330 namespace: kserve-ci-e2e-test resourceVersion: "26860" uid: 129c20ea-a176-4365-bbc0-d958c8d155da reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:06:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-73f94 namespace: kserve-ci-e2e-test resourceVersion: "26578" uid: 4dc37f10-f44c-4ff8-b938-5c84a6cdf8d8 kind: Event lastTimestamp: "2026-07-08T17:06:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:06:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:06:29Z" name: switch-graph-73f94.18c05f6d140ab573 namespace: kserve-ci-e2e-test resourceVersion: "26861" uid: cc89ead7-5ed3-42c3-bc5c-9e2b9e52b9fd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:45:39.482227Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-8ceba-5774d86bd6-r5cbl namespace: kserve-ci-e2e-test resourceVersion: "16231" uid: 3cfc49f7-1e6d-4d50-b91d-d223d148ac28 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-8ceba-5774d86bd6-r5cbl to ip-10-0-132-182.ec2.internal metadata: creationTimestamp: "2026-07-08T16:45:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:45:39Z" name: switch-graph-8ceba-5774d86bd6-r5cbl.18c05e4d856591ed namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: db3599dc-0e02-484f-a88f-edea038d2384 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:39Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-8ceba-5774d86bd6-r5cbl namespace: kserve-ci-e2e-test resourceVersion: "16239" uid: 3cfc49f7-1e6d-4d50-b91d-d223d148ac28 kind: Event lastTimestamp: "2026-07-08T16:45:39Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:45:39Z" name: switch-graph-8ceba-5774d86bd6-r5cbl.18c05e4d9f1150a2 namespace: kserve-ci-e2e-test resourceVersion: "16253" uid: 18a35be7-716b-432b-ba1c-8a4873c48d9e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8ceba} kind: Pod name: switch-graph-8ceba-5774d86bd6-r5cbl namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 3cfc49f7-1e6d-4d50-b91d-d223d148ac28 kind: Event lastTimestamp: "2026-07-08T16:45:39Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:86bfaabcc176433317a84dc02e835b9657d32290b963c2e4ae8312aa72bd9d64" already present on machine metadata: creationTimestamp: "2026-07-08T16:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:39Z" name: switch-graph-8ceba-5774d86bd6-r5cbl.18c05e4da06261aa namespace: kserve-ci-e2e-test resourceVersion: "16255" uid: 9001b1db-b893-4e0d-96f8-282ca74437aa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8ceba} kind: Pod name: switch-graph-8ceba-5774d86bd6-r5cbl namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 3cfc49f7-1e6d-4d50-b91d-d223d148ac28 kind: Event lastTimestamp: "2026-07-08T16:45:39Z" message: 'Created container: switch-graph-8ceba' metadata: creationTimestamp: "2026-07-08T16:45:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:40Z" name: switch-graph-8ceba-5774d86bd6-r5cbl.18c05e4da424fb5b namespace: kserve-ci-e2e-test resourceVersion: "16256" uid: 713889aa-f5bd-418f-a53a-b22aa807160f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8ceba} kind: Pod name: switch-graph-8ceba-5774d86bd6-r5cbl namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 3cfc49f7-1e6d-4d50-b91d-d223d148ac28 kind: Event lastTimestamp: "2026-07-08T16:45:40Z" message: Started container switch-graph-8ceba metadata: creationTimestamp: "2026-07-08T16:45:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:40Z" name: switch-graph-8ceba-5774d86bd6-r5cbl.18c05e4da48ed0c8 namespace: kserve-ci-e2e-test resourceVersion: "16257" uid: e1e0528e-559f-4b52-aa4e-ebac87d86905 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8ceba} kind: Pod name: switch-graph-8ceba-5774d86bd6-r5cbl namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 3cfc49f7-1e6d-4d50-b91d-d223d148ac28 kind: Event lastTimestamp: "2026-07-08T16:53:54Z" message: Stopping container switch-graph-8ceba metadata: creationTimestamp: "2026-07-08T16:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:54Z" name: switch-graph-8ceba-5774d86bd6-r5cbl.18c05ec0aff7a030 namespace: kserve-ci-e2e-test resourceVersion: "19746" uid: 4c480fcf-f187-42e2-aee8-3a5f79b39873 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:53:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8ceba} kind: Pod name: switch-graph-8ceba-5774d86bd6-r5cbl namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 3cfc49f7-1e6d-4d50-b91d-d223d148ac28 kind: Event lastTimestamp: "2026-07-08T16:54:21Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T16:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:21Z" name: switch-graph-8ceba-5774d86bd6-r5cbl.18c05ec127363ca8 namespace: kserve-ci-e2e-test resourceVersion: "20085" uid: aaccc9eb-05cb-44d2-a22a-01baa75f8a91 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-8ceba-5774d86bd6 namespace: kserve-ci-e2e-test resourceVersion: "16228" uid: f2e91082-331b-43da-b7f0-d84a7d605885 kind: Event lastTimestamp: "2026-07-08T16:45:39Z" message: 'Created pod: switch-graph-8ceba-5774d86bd6-r5cbl' metadata: creationTimestamp: "2026-07-08T16:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:45:39Z" name: switch-graph-8ceba-5774d86bd6.18c05e4d84f4b454 namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: cb721b6d-91f6-46d8-a8eb-c743f92c9ebb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-8ceba namespace: kserve-ci-e2e-test resourceVersion: "16226" uid: 7376af41-ea67-41a9-8696-afed0ef24dbd kind: Event lastTimestamp: "2026-07-08T16:45:39Z" message: Scaled up replica set switch-graph-8ceba-5774d86bd6 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:45:39Z" name: switch-graph-8ceba.18c05e4d83f3e9a1 namespace: kserve-ci-e2e-test resourceVersion: "16229" uid: 79e47aa5-8498-488a-9ebd-d72d427014af reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-8ceba namespace: kserve-ci-e2e-test resourceVersion: "16244" uid: 15ee8010-54ab-433d-9e3f-db3970d1a2f2 kind: Event lastTimestamp: "2026-07-08T16:45:39Z" message: InferenceGraph [switch-graph-8ceba] is Ready metadata: creationTimestamp: "2026-07-08T16:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:45:39Z" name: switch-graph-8ceba.18c05e4d898a5cd5 namespace: kserve-ci-e2e-test resourceVersion: "16246" uid: 58b8addb-aa69-4c0a-8d4b-f7a3e49cd6d5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:45:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-8ceba namespace: kserve-ci-e2e-test resourceVersion: "16227" uid: f10f3a03-4519-4d95-afe5-90f085b3bd36 kind: Event lastTimestamp: "2026-07-08T16:46:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:45:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:09Z" name: switch-graph-8ceba.18c05e510275d2e4 namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: 31d708fb-304a-436c-98de-6889be2e3658 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:45:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-8ceba namespace: kserve-ci-e2e-test resourceVersion: "16227" uid: f10f3a03-4519-4d95-afe5-90f085b3bd36 kind: Event lastTimestamp: "2026-07-08T16:46:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:45:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:09Z" name: switch-graph-8ceba.18c05e5102de562b namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: 63d40d96-4599-46bd-9dac-996a2e1f793b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:44:29.020295Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-ed4fa-6678548b58-ft22c namespace: kserve-ci-e2e-test resourceVersion: "15220" uid: 1be8a854-ec39-4f0d-bf33-067e97ba024d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-ed4fa-6678548b58-ft22c to ip-10-0-132-182.ec2.internal metadata: creationTimestamp: "2026-07-08T16:44:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:44:29Z" name: switch-graph-ed4fa-6678548b58-ft22c.18c05e3d1d89cd59 namespace: kserve-ci-e2e-test resourceVersion: "15226" uid: 1b821254-36c5-4766-8e78-bc7b967c939a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76ffc46cff-hs7js source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:29Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-ed4fa-6678548b58-ft22c namespace: kserve-ci-e2e-test resourceVersion: "15230" uid: 1be8a854-ec39-4f0d-bf33-067e97ba024d kind: Event lastTimestamp: "2026-07-08T16:44:29Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:44:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:44:29Z" name: switch-graph-ed4fa-6678548b58-ft22c.18c05e3d36ba3f24 namespace: kserve-ci-e2e-test resourceVersion: "15241" uid: ff2164cf-4010-4470-8ea7-7582c38f2c6e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ed4fa} kind: Pod name: switch-graph-ed4fa-6678548b58-ft22c namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 1be8a854-ec39-4f0d-bf33-067e97ba024d kind: Event lastTimestamp: "2026-07-08T16:44:29Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:86bfaabcc176433317a84dc02e835b9657d32290b963c2e4ae8312aa72bd9d64" metadata: creationTimestamp: "2026-07-08T16:44:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:29Z" name: switch-graph-ed4fa-6678548b58-ft22c.18c05e3d380b3a6f namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: 8d860126-af1d-433e-b9de-c8ca39f79b79 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ed4fa} kind: Pod name: switch-graph-ed4fa-6678548b58-ft22c namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 1be8a854-ec39-4f0d-bf33-067e97ba024d kind: Event lastTimestamp: "2026-07-08T16:44:31Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:86bfaabcc176433317a84dc02e835b9657d32290b963c2e4ae8312aa72bd9d64" in 2.463s (2.463s including waiting). Image size: 218072711 bytes.' metadata: creationTimestamp: "2026-07-08T16:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:31Z" name: switch-graph-ed4fa-6678548b58-ft22c.18c05e3dcadfec6b namespace: kserve-ci-e2e-test resourceVersion: "15257" uid: a4431311-dcf5-4231-ab25-91fbffd76260 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ed4fa} kind: Pod name: switch-graph-ed4fa-6678548b58-ft22c namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 1be8a854-ec39-4f0d-bf33-067e97ba024d kind: Event lastTimestamp: "2026-07-08T16:44:31Z" message: 'Created container: switch-graph-ed4fa' metadata: creationTimestamp: "2026-07-08T16:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:31Z" name: switch-graph-ed4fa-6678548b58-ft22c.18c05e3dce623703 namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: 097c834d-859b-4df9-ae2e-bb5e66509895 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ed4fa} kind: Pod name: switch-graph-ed4fa-6678548b58-ft22c namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 1be8a854-ec39-4f0d-bf33-067e97ba024d kind: Event lastTimestamp: "2026-07-08T16:44:31Z" message: Started container switch-graph-ed4fa metadata: creationTimestamp: "2026-07-08T16:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:31Z" name: switch-graph-ed4fa-6678548b58-ft22c.18c05e3dcec2d656 namespace: kserve-ci-e2e-test resourceVersion: "15259" uid: 7d2728b0-08aa-4089-8fa5-f8fdf05117a7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ed4fa} kind: Pod name: switch-graph-ed4fa-6678548b58-ft22c namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 1be8a854-ec39-4f0d-bf33-067e97ba024d kind: Event lastTimestamp: "2026-07-08T16:44:39Z" message: Stopping container switch-graph-ed4fa metadata: creationTimestamp: "2026-07-08T16:44:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:39Z" name: switch-graph-ed4fa-6678548b58-ft22c.18c05e3f7cf45743 namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 1302015a-3000-4cf3-8a23-76b61ad2367f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:44:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ed4fa} kind: Pod name: switch-graph-ed4fa-6678548b58-ft22c namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 1be8a854-ec39-4f0d-bf33-067e97ba024d kind: Event lastTimestamp: "2026-07-08T16:45:08Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T16:44:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:08Z" name: switch-graph-ed4fa-6678548b58-ft22c.18c05e408fc424ba namespace: kserve-ci-e2e-test resourceVersion: "15743" uid: 6b6da7f7-11a3-4f68-8474-eb98cd7af00c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-182.ec2.internal source: component: kubelet host: ip-10-0-132-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-ed4fa-6678548b58 namespace: kserve-ci-e2e-test resourceVersion: "15217" uid: 664f1e1b-de22-41ed-bf67-82cdf39279bf kind: Event lastTimestamp: "2026-07-08T16:44:29Z" message: 'Created pod: switch-graph-ed4fa-6678548b58-ft22c' metadata: creationTimestamp: "2026-07-08T16:44:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:44:29Z" name: switch-graph-ed4fa-6678548b58.18c05e3d1d14c399 namespace: kserve-ci-e2e-test resourceVersion: "15222" uid: 5fb481ed-7a1e-460a-9c4e-a9eb70264ad7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-ed4fa namespace: kserve-ci-e2e-test resourceVersion: "15215" uid: 2b65f5b2-c83a-4120-8236-b612580f0a19 kind: Event lastTimestamp: "2026-07-08T16:44:28Z" message: Scaled up replica set switch-graph-ed4fa-6678548b58 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:44:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:44:28Z" name: switch-graph-ed4fa.18c05e3d1c0c5504 namespace: kserve-ci-e2e-test resourceVersion: "15218" uid: aeb0beb8-e9ee-4fd0-a801-0b168af72b35 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:29Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-ed4fa namespace: kserve-ci-e2e-test resourceVersion: "15232" uid: 0acbdd1a-ff55-4587-82e4-cd64b0e3e433 kind: Event lastTimestamp: "2026-07-08T16:44:29Z" message: InferenceGraph [switch-graph-ed4fa] is Ready metadata: creationTimestamp: "2026-07-08T16:44:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:44:29Z" name: switch-graph-ed4fa.18c05e3d204c8ba3 namespace: kserve-ci-e2e-test resourceVersion: "15235" uid: dfb26a45-0e2c-496f-9a31-de01b0c43210 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal kind: EventList metadata: resourceVersion: "33749"